diff --git a/codelets/Makefile.defs b/codelets/Makefile.defs index 27ebf8e..76d57ce 100644 --- a/codelets/Makefile.defs +++ b/codelets/Makefile.defs @@ -76,12 +76,18 @@ ifeq ($(origin CC),default) CC := clang endif +# `-target bpf` does not search the multiarch include directory, where glibc +# keeps bits/*.h on Debian/Ubuntu (/usr/include/aarch64-linux-gnu on arm64), so +# fails there. Add it explicitly, as jbpf's own test codelets do. +SYS_INC := $(addprefix -I,$(wildcard /usr/include/$(shell uname -m)-linux-gnu)) + INC := -I$(JBPF_OUT_DIR)/inc \ -I$(JBPF_DIR)/src/core \ -I$(JBPF_DIR)/src/common \ -I$(OCUDU_JANUS_INC) \ -I$(OCUDU_SPECS_INC) \ - -I$(CONTRACT_INC) + -I$(CONTRACT_INC) \ + $(SYS_INC) # -target bpf: codelets are eBPF objects, loaded by ubpf in the gNB. # diff --git a/codelets/include/e3_slot_convert.h b/codelets/include/e3_slot_convert.h index 9306a2f..4aa8700 100644 --- a/codelets/include/e3_slot_convert.h +++ b/codelets/include/e3_slot_convert.h @@ -26,7 +26,8 @@ * which is why the caller validates it. * * The transform is bf16 -> fp32 (a 16-bit shift, free) -> x scale -> - * fp32 -> fp16 (needs rounding + range clamp, hence F16C). + * fp32 -> fp16 (needs rounding + range clamp, hence F16C on x86 and the + * AdvSIMD FCVTN instruction on aarch64; both round to nearest-even). * --------------------------------------------------------------------------- */ @@ -40,6 +41,16 @@ #include #endif +/* The vector path this build uses. NEON is part of every aarch64 CPU, so it is + * always on there; E3_NO_NEON hides it, for the microbenchmark's scalar + * baseline. */ +#if defined(__AVX2__) && defined(__F16C__) +#define E3_CONVERT_AVX2 1 +#elif defined(__aarch64__) && !defined(E3_NO_NEON) +#define E3_CONVERT_NEON 1 +#include +#endif + /* * VECTOR PATH REQUIRED. * @@ -56,12 +67,10 @@ * E3_ALLOW_SCALAR_CONVERT to opt out (host tooling, non-x86, or the * microbenchmark's deliberate scalar baseline). */ -#if !defined(E3_ALLOW_SCALAR_CONVERT) -#if !(defined(__AVX2__) && defined(__F16C__)) && !defined(__aarch64__) -#error "e3_slot_convert.h: no vector path. Compile with -mavx2 -mf16c (or -march=native). \ +#if !defined(E3_ALLOW_SCALAR_CONVERT) && !defined(E3_CONVERT_AVX2) && !defined(E3_CONVERT_NEON) +#error "e3_slot_convert.h: no vector path. Compile with -mavx2 -mf16c (or -march=native) on x86. \ Define E3_ALLOW_SCALAR_CONVERT only if a scalar convert in the RX thread is genuinely acceptable." #endif -#endif namespace e3_convert { @@ -105,6 +114,15 @@ bf16_to_fp16_scalar(std::uint16_t bf16, float scale) __m128 ss = _mm_set_ss(u32.f * scale); __m128i ph = _mm_cvtps_ph(ss, _MM_FROUND_TO_NEAREST_INT); return static_cast(_mm_extract_epi16(ph, 0)); +#elif defined(E3_CONVERT_NEON) + /* The same instruction as the vector path, so the two agree bit for bit. */ + union { + std::uint32_t u; + float f; + } u32; + u32.u = static_cast(bf16) << 16; + const float16x4_t h = vcvt_f16_f32(vdupq_n_f32(u32.f * scale)); + return vget_lane_u16(vreinterpret_u16_f16(h), 0); #else union { std::uint32_t u; @@ -153,13 +171,14 @@ bf16_to_fp16_scalar(std::uint16_t bf16, float scale) /* * Convert one antenna's worth of bf16 to fp16, applying `scale`. * - * `n_u16` MUST be a multiple of 8 on the AVX2 path. For 273 PRB / 14 symbols it - * is 91,728 = 8 * 11,466, so there is no tail; asserted by the caller. + * The vector paths take 8 samples per iteration; a remainder goes through the + * scalar function. For 273 PRB / 14 symbols n_u16 is 91,728 = 8 * 11,466, so + * there is no tail. */ inline void convert_ant(std::uint16_t* dst, const std::uint16_t* src, std::size_t n_u16, float scale) { -#if defined(__AVX2__) && defined(__F16C__) +#if defined(E3_CONVERT_AVX2) /* 8 bf16 -> 8 fp16 per iteration: * load 8 bf16 (16 B) -> zero-extend to 8x u32 -> shift left 16 (bf16 * promoted to fp32) -> multiply by scale -> cvtps_ph -> store 16 B. */ @@ -177,6 +196,25 @@ convert_ant(std::uint16_t* dst, const std::uint16_t* src, std::size_t n_u16, flo for (; i < n_u16; ++i) { /* tail; empty for 273 PRB */ dst[i] = bf16_to_fp16_scalar(src[i], scale); } +#elif defined(E3_CONVERT_NEON) + /* 8 bf16 -> 8 fp16 per iteration: + * load 8 bf16 (16 B) -> widen each half to 4x u32 shifted left 16 (bf16 + * promoted to fp32) -> multiply by scale -> FCVTN/FCVTN2 to 8 fp16 (round + * to nearest-even, as _MM_FROUND_TO_NEAREST_INT above) -> store 16 B. */ + const float32x4_t v_scale = vdupq_n_f32(scale); + std::size_t i = 0; + for (; i + 8 <= n_u16; i += 8) { + const uint16x8_t b16 = vld1q_u16(src + i); + const uint32x4_t lo = vshll_n_u16(vget_low_u16(b16), 16); + const uint32x4_t hi = vshll_high_n_u16(b16, 16); + const float32x4_t flo = vmulq_f32(vreinterpretq_f32_u32(lo), v_scale); + const float32x4_t fhi = vmulq_f32(vreinterpretq_f32_u32(hi), v_scale); + const float16x8_t ph = vcvt_high_f16_f32(vcvt_f16_f32(flo), fhi); + vst1q_u16(dst + i, vreinterpretq_u16_f16(ph)); + } + for (; i < n_u16; ++i) { /* tail; empty for 273 PRB */ + dst[i] = bf16_to_fp16_scalar(src[i], scale); + } #else for (std::size_t i = 0; i < n_u16; ++i) { dst[i] = bf16_to_fp16_scalar(src[i], scale); @@ -214,13 +252,26 @@ convert_slot(std::uint16_t* dst, const std::uint16_t* src, const RowGeometry& ge inline constexpr bool has_vector_path() { -#if defined(__AVX2__) && defined(__F16C__) +#if defined(E3_CONVERT_AVX2) || defined(E3_CONVERT_NEON) return true; #else return false; #endif } +/* The same, as the name to put in that log line. */ +inline constexpr const char* +vector_path_name() +{ +#if defined(E3_CONVERT_AVX2) + return "AVX2+F16C"; +#elif defined(E3_CONVERT_NEON) + return "NEON"; +#else + return "SCALAR(!!)"; +#endif +} + } // namespace e3_convert #endif /* E3_SLOT_CONVERT_H */ diff --git a/codelets/verifier/bench_slot_convert.cpp b/codelets/verifier/bench_slot_convert.cpp index 516735c..8ea8933 100644 --- a/codelets/verifier/bench_slot_convert.cpp +++ b/codelets/verifier/bench_slot_convert.cpp @@ -17,6 +17,7 @@ * * Build (must be explicit about the ISA — that is the point): * g++ -O2 -std=c++17 -mavx2 -mf16c -I../include bench_slot_convert.cpp -o bench_slot_convert + * g++ -O2 -std=c++17 -I../include bench_slot_convert.cpp -o bench_slot_convert (aarch64: NEON) * g++ -O2 -std=c++17 -DE3_ALLOW_SCALAR_CONVERT -DBENCH_FORCE_SCALAR \ * -I../include bench_slot_convert.cpp -o bench_slot_convert_scalar */ @@ -34,6 +35,7 @@ /* Deliberately hide the vector path so we can price the fallback. */ #undef __AVX2__ #undef __F16C__ +#define E3_NO_NEON 1 #define E3_ALLOW_SCALAR_CONVERT 1 #endif @@ -118,7 +120,7 @@ main(int argc, char** argv) int iters = (argc > 1) ? std::atoi(argv[1]) : 200; std::printf("bench_slot_convert — cbf16 -> fp16, 14 symbols x 3276 subcarriers\n"); - std::printf("vector path compiled in: %s\n", e3_convert::has_vector_path() ? "YES (AVX2+F16C)" : "NO (SCALAR)"); + std::printf("vector path compiled in: %s\n", e3_convert::vector_path_name()); std::printf("iterations per config: %d (median / p95 reported)\n", iters); std::printf("throughput counts bytes READ + WRITTEN\n\n");