From b58f2b42660d3d1e6d56e6e08e4cd2cd8b7742d5 Mon Sep 17 00:00:00 2001 From: Benjamin Meyer Date: Fri, 18 Sep 2026 20:42:01 +0200 Subject: [PATCH] Remove ImageToPatchEmbedder (to specific for core) --- README.md | 2 +- .../embedder/ImageToPatchEmbedder.scala | 52 --------------- .../embedder/ImageToPatchEmbedderSuite.scala | 63 ------------------- mdocs/README.md | 2 +- 4 files changed, 2 insertions(+), 117 deletions(-) delete mode 100644 core/src/main/scala/deepwit/embedder/ImageToPatchEmbedder.scala delete mode 100644 core/src/test/scala/deepwit/embedder/ImageToPatchEmbedderSuite.scala diff --git a/README.md b/README.md index 7ca4775..32aa9c1 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The user code composes these core modules into custom architectures given the us | `deepwit.cnn` | `LinearConv2DLayer`, `AffineConv2DLayer`, both transpose variants, `MaxPool2DLayer` | | `deepwit.attention` | scaled-dot-product scores, full / causal / custom masking, multi-head (fused and unfused), self-attention, a readable reference implementation | | `deepwit.transformer` | `TransformerBlock`, `CrossTransformerBlock` — the residual skeleton, with the mixers left open | -| `deepwit.embedder` | `VocabularyEmbedder` (with tied unembedding), `LearnedAbsolutePositionalInjector`, `ImageToPatchEmbedder`, `PositionalEncoding.sinusoidal2D` | +| `deepwit.embedder` | `VocabularyEmbedder` (with tied unembedding), `LearnedAbsolutePositionalInjector`, `PositionalEncoding.sinusoidal2D` | | `deepwit.normalization` | `LayerNorm`, `RMSNorm` | | `deepwit.activation` | `sigmoid`, `relu`, `gelu`, `softmax` | | `deepwit.loss` | `CategoricalCrossEntropy`, `BernoulliCrossEntropy`, `BinaryCrossEntropy`, `SquaredError`, `AbsoluteError`, `Huber` | diff --git a/core/src/main/scala/deepwit/embedder/ImageToPatchEmbedder.scala b/core/src/main/scala/deepwit/embedder/ImageToPatchEmbedder.scala deleted file mode 100644 index 3ffe6ac..0000000 --- a/core/src/main/scala/deepwit/embedder/ImageToPatchEmbedder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package deepwit.embedder - -import dimwit.* -import deepwit.cnn.AffineConv2DLayer -import deepwit.embedder.PositionalEncoding.sinusoidal2D -import dimwit.Label as Λ - -/** Cuts an image into non-overlapping patches and embeds every patch into a sequence element. - * - * The patches are produced by a strided convolution whose stride equals the kernel size, and are - * enriched with a 2D sinusoidal positional encoding before being flattened into a sequence. - * - * @tparam Width The axis label for the image width. - * @tparam Height The axis label for the image height. - * @tparam Channel The axis label for the image channels. - * @tparam PatchEmbedding The axis label for the patch embedding space. - * @tparam V The floating-point scalar type of the tensor elements. - * @param params The learnable parameters. - */ -class ImageToPatchEmbedder[ - Width: Λ, - Height: Λ, - Channel: Λ, - PatchEmbedding: Λ, - V: IsFloating -]( - params: ImageToPatchEmbedder.Params[Width, Height, Channel, PatchEmbedding, V] -) extends (Tensor3[Width, Height, Channel, V] => Tensor2[Width |*| Height, PatchEmbedding, V]): - - private val convLayer = - val kernelShape = params.conv.kernel.shape - val kernelSize = (kernelShape.extent(Axis[Width]), kernelShape.extent(Axis[Height])) - AffineConv2DLayer(params.conv, stride = kernelSize) - - override def apply(img: Tensor3[Width, Height, Channel, V]): Tensor2[Width |*| Height, PatchEmbedding, V] = - val patches = convLayer(img) - val patchesPos = patches + sinusoidal2D(patches.shape) - patchesPos.flatten((Axis[Width], Axis[Height])) - -object ImageToPatchEmbedder: - - case class Params[PatchWidth, PatchHeight, Channel, PatchEmbedding, V]( - conv: AffineConv2DLayer.Params[PatchWidth, PatchHeight, Channel, PatchEmbedding, V] - ) - - object Params: - - def init[PatchWidth: Λ, PatchHeight: Λ, Channel: Λ, PatchEmbedding: Λ, V: IsFloating](patchWidthExtent: AxisExtent[PatchWidth], patchHeightExtent: AxisExtent[PatchHeight], channelExtent: AxisExtent[Channel], embeddingExtent: AxisExtent[PatchEmbedding], key: Key, vtype: VType[V] = VType[Float32]): Params[PatchWidth, PatchHeight, Channel, PatchEmbedding, V] = - xavierUniform(patchWidthExtent, patchHeightExtent, channelExtent, embeddingExtent, key, vtype) - - def xavierUniform[PatchWidth: Λ, PatchHeight: Λ, Channel: Λ, PatchEmbedding: Λ, V: IsFloating](patchWidthExtent: AxisExtent[PatchWidth], patchHeightExtent: AxisExtent[PatchHeight], channelExtent: AxisExtent[Channel], embeddingExtent: AxisExtent[PatchEmbedding], key: Key, vtype: VType[V] = VType[Float32]): Params[PatchWidth, PatchHeight, Channel, PatchEmbedding, V] = - Params(conv = AffineConv2DLayer.Params.xavierUniform(patchWidthExtent, patchHeightExtent, channelExtent, embeddingExtent, key, vtype)) diff --git a/core/src/test/scala/deepwit/embedder/ImageToPatchEmbedderSuite.scala b/core/src/test/scala/deepwit/embedder/ImageToPatchEmbedderSuite.scala deleted file mode 100644 index 6083e2e..0000000 --- a/core/src/test/scala/deepwit/embedder/ImageToPatchEmbedderSuite.scala +++ /dev/null @@ -1,63 +0,0 @@ -package deepwit.embedder - -import deepwit.* -import dimwit.* -import org.scalatest.matchers.should.Matchers -import org.scalatest.funspec.AnyFunSpec - -class ImageToPatchEmbedderSuite extends AnyFunSpec with Matchers: - - trait Width derives Label - trait Height derives Label - trait Channel derives Label - trait PatchEmbedding derives Label - - private val patchExtent = 2 - private val embeddingExtent = Axis[PatchEmbedding] -> 4 - - private def embedder = - val params = ImageToPatchEmbedder.Params.xavierUniform( - Axis[Width] -> patchExtent, - Axis[Height] -> patchExtent, - Axis[Channel] -> 1, - embeddingExtent, - Random.Key(42) - ) - ImageToPatchEmbedder(params) - - private def image = Tensor(Shape(Axis[Width] -> 8, Axis[Height] -> 8, Axis[Channel] -> 1), VType[Float32]).fill(0.5f) - - describe("ImageToPatchEmbedder"): - - it("produces one embedded patch per patch of the image"): - // An 8x8 image cut into 2x2 patches yields a 4x4 grid, flattened to 16 sequence elements. - val patches = embedder(image) - patches.shape(Axis[Width |*| Height]) shouldBe 16 - patches.shape(Axis[PatchEmbedding]) shouldBe 4 - - it("is deterministic for the same parameters"): - val fixed = embedder - fixed(image) should approxEqual(fixed(image), 0f) - - it("distinguishes patches through the positional encoding"): - // The image is constant, so any difference between patches comes from the 2D encoding. - val patches = embedder(image) - val first = patches.slice(Axis[Width |*| Height].at(0)) - val last = patches.slice(Axis[Width |*| Height].at(15)) - (first - last).abs.max.item should be > 1e-3f - - describe("ImageToPatchEmbedder.Params"): - - it("xavierUniform builds a kernel of the patch shape"): - val params = ImageToPatchEmbedder.Params.xavierUniform( - Axis[Width] -> 4, - Axis[Height] -> 4, - Axis[Channel] -> 3, - embeddingExtent, - Random.Key(42) - ) - params.conv.kernel.shape(Axis[Width]) shouldBe 4 - params.conv.kernel.shape(Axis[Height]) shouldBe 4 - params.conv.kernel.shape(Axis[Channel]) shouldBe 3 - params.conv.kernel.shape(Axis[PatchEmbedding]) shouldBe 4 - params.conv.bias should approxEqual(Tensor(Shape1(embeddingExtent)).fill(0f), 1e-6f) diff --git a/mdocs/README.md b/mdocs/README.md index 9050bc9..e703cf5 100644 --- a/mdocs/README.md +++ b/mdocs/README.md @@ -173,7 +173,7 @@ The user code composes these core modules into custom architectures given the us | `deepwit.cnn` | `LinearConv2DLayer`, `AffineConv2DLayer`, both transpose variants, `MaxPool2DLayer` | | `deepwit.attention` | scaled-dot-product scores, full / causal / custom masking, multi-head (fused and unfused), self-attention, a readable reference implementation | | `deepwit.transformer` | `TransformerBlock`, `CrossTransformerBlock` — the residual skeleton, with the mixers left open | -| `deepwit.embedder` | `VocabularyEmbedder` (with tied unembedding), `LearnedAbsolutePositionalInjector`, `ImageToPatchEmbedder`, `PositionalEncoding.sinusoidal2D` | +| `deepwit.embedder` | `VocabularyEmbedder` (with tied unembedding), `LearnedAbsolutePositionalInjector`, `PositionalEncoding.sinusoidal2D` | | `deepwit.normalization` | `LayerNorm`, `RMSNorm` | | `deepwit.activation` | `sigmoid`, `relu`, `gelu`, `softmax` | | `deepwit.loss` | `CategoricalCrossEntropy`, `BernoulliCrossEntropy`, `BinaryCrossEntropy`, `SquaredError`, `AbsoluteError`, `Huber` |