Model and Feature Extractor

config.json

@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "BirdMAEModel"
+  ],
+  "attn_drop_rate": 0.0,
+  "auto_map": {
+    "AutoConfig": "config.BirdMAEConfig",
+    "AutoModel": "model.BirdMAEModel"
+  },
+  "depth": 12,
+  "drop_path_rate": 0.0,
+  "drop_rate": 0.0,
+  "embed_dim": 768,
+  "img_size_x": 512,
+  "img_size_y": 128,
+  "in_chans": 1,
+  "init_values": null,
+  "mlp_ratio": 4.0,
+  "norm_layer_eps": 1e-06,
+  "num_heads": 12,
+  "num_patches": 256,
+  "num_patches_x": 32,
+  "num_patches_y": 8,
+  "num_tokens": 257,
+  "patch_size": 16,
+  "pos_drop_rate": 0.0,
+  "pos_trainable": false,
+  "proj_drop_rate": 0.0,
+  "qk_norm": false,
+  "qkv_bias": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.38.0"
+}

config.py

@@ -0,0 +1,55 @@
+from transformers import PretrainedConfig
+import torch.nn as nn  # For norm_layer type
+
+
+class BirdMAEConfig(PretrainedConfig):
+    _auto_class = "AutoConfig"
+
+    def __init__(
+            self,
+            img_size_x=512,  # From provided config
+            img_size_y=128,  # From provided config
+            patch_size=16,  # From provided config
+            in_chans=1,  # From provided config
+            embed_dim=768,  # From provided config
+            depth=12,  # From provided config
+            num_heads=12,  # From provided config
+            mlp_ratio=4.0,  # From provided config
+            pos_trainable=False,  # From provided config
+            qkv_bias: bool = True,
+            qk_norm: bool = False,
+            init_values: float = None,
+            drop_rate=0.0,  # Not explicitly in your MAE_Encoder init, but Block has it
+            # attn_drop_rate=0.0,  # Not explicitly in your MAE_Encoder init, but Block has it
+            # drop_path_rate=0.0,  # Not explicitly in your MAE_Encoder init, but Block has it
+            norm_layer_eps=1e-6,  # Default for nn.LayerNorm
+            #cls_token=True,  # Your MAE_Encoder uses self.cls_token
+            **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.img_size_x = img_size_x
+        self.img_size_y = img_size_y
+        self.patch_size = patch_size
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+        self.depth = depth
+        self.num_heads = num_heads
+        self.mlp_ratio = mlp_ratio
+        self.pos_trainable = pos_trainable
+
+        self.qkv_bias = qkv_bias
+        self.qk_norm = qk_norm
+        self.init_values = init_values
+        self.drop_rate = drop_rate
+        self.pos_drop_rate = drop_rate
+        self.attn_drop_rate = drop_rate
+        self.drop_path_rate = drop_rate
+        self.proj_drop_rate = drop_rate
+        self.norm_layer_eps = norm_layer_eps
+
+        # Calculated properties (useful for initializing the model)
+        self.num_patches_x = img_size_x // patch_size
+        self.num_patches_y = img_size_y // patch_size
+        self.num_patches = self.num_patches_x * self.num_patches_y
+        self.num_tokens = self.num_patches + 1

feature_extractor.py

@@ -0,0 +1,132 @@
+from transformers import SequenceFeatureExtractor
+from transformers.tokenization_utils_base import BatchEncoding
+from transformers.feature_extraction_utils import BatchFeature
+from torchaudio.compliance.kaldi import fbank
+import torch
+import numpy as np
+import torch.nn.functional as F
+
+from typing import Union, List
+from transformers.utils import PaddingStrategy
+
+
+class BirdMAEFeatureExtractor(SequenceFeatureExtractor):
+    _auto_class = "AutoFeatureExtractor"
+    model_input_names = ["input_values"]
+
+    def __init__(
+            self,
+            # process waveform
+            feature_size: int = 1,
+            sampling_rate: int = 32_000,
+            padding_value: float = 0.0,
+            return_attention_mask: bool = True,
+
+            # fbank
+            htk_compat: bool = True,
+            use_energy: bool = False,
+            window_type: str = "hanning",
+            num_mel_bins: int = 128,
+            dither: float = 0.0,
+            frame_shift: int = 10,
+
+            # pad and normalize
+            target_length: int = 512,
+            mean: float = -7.2,
+            std: float = 4.43,
+
+            **kwargs
+    ):
+        super().__init__(feature_size, sampling_rate, padding_value, **kwargs)
+        # squence FE
+        self.feature_size = feature_size
+        self.sampling_rate = sampling_rate
+        self.padding_value = padding_value
+        self.return_attention_mask = return_attention_mask
+
+        # fbank
+        self.htk_compat = htk_compat
+        self.use_energy = use_energy
+        self.window_type = window_type
+        self.num_mel_bins = num_mel_bins
+        self.dither = dither
+        self.frame_shift = frame_shift
+
+        # pad and normalize
+        self.target_length = target_length
+        self.mean = mean
+        self.std = std
+
+    def __call__(self,
+                 waveform_batch: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
+                 padding: Union[bool, str, PaddingStrategy] = "max_length",
+                 max_length: int | None = None,
+                 truncation: bool = True,
+                 return_tensors: str = "pt"
+                 ):
+
+        if isinstance(waveform_batch, (list, np.ndarray)) and not isinstance(waveform_batch[0], (list, np.ndarray)):
+            waveform_batch = [waveform_batch]
+
+
+        waveform_batch = self._process_waveforms(waveform_batch, padding, truncation)
+
+        fbank_features = self._compute_fbank_features(waveform_batch["input_values"])
+
+        fbank_features = self._pad_and_normalize(fbank_features)
+
+        return fbank_features
+
+    def _process_waveforms(self,
+                           waveforms,
+                           padding: bool | str,
+                           truncation: bool):
+        clip_duration = 5 # TODO this is the clip duration used in training
+        max_length = int(int(self.sampling_rate) * clip_duration)
+        waveform_encoded = BatchFeature({"input_values": waveforms})
+
+        waveform_batch = self.pad(
+            waveform_encoded,
+            padding=padding,
+            max_length=max_length,
+            truncation=truncation,
+            return_attention_mask=self.return_attention_mask
+        )
+        waveform_batch["input_values"] = torch.tensor(
+            waveform_batch["input_values"])
+        attention_mask = waveform_batch.get("attention_mask")
+
+        if attention_mask is not None:
+            waveform_batch["attention_mask"] = attention_mask
+
+        # add std
+        waveform_batch["input_values"] = waveform_batch["input_values"] - waveform_batch["input_values"].mean(axis=1, keepdims=True)
+        #waveform_batch["input_values"] = (waveform_batch["input_values"] - waveform_batch["input_values"].mean(axis=1, keepdims=True)) / (waveform_batch["input_values"].std(axis=1, keepdims=True) + 1e-8)
+        return waveform_batch
+
+    def _compute_fbank_features(self, waveforms):
+        fbank_features = [
+            fbank(
+                waveform.unsqueeze(0),
+                htk_compat=self.htk_compat,
+                sample_frequency=self.sampling_rate,
+                use_energy=self.use_energy,
+                window_type=self.window_type,
+                num_mel_bins=self.num_mel_bins,
+                dither=self.dither,
+                frame_shift=self.frame_shift
+            )
+            for waveform in waveforms
+        ]
+        return torch.stack(fbank_features)
+
+    def _pad_and_normalize(self, fbank_features):
+        difference = self.target_length - fbank_features[0].shape[0]
+        min_value = fbank_features.min()
+
+        if self.target_length > fbank_features.shape[0]:
+            padding = (0, 0, 0, difference)
+            fbank_features = F.pad(fbank_features, padding, value=min_value.item())
+
+        fbank_features = (fbank_features - self.mean) / (self.std * 2)
+        return fbank_features

model.py

@@ -0,0 +1,452 @@
+import torch
+import torch.nn as nn
+from torch import Tensor
+import math
+import numpy as np
+
+from transformers import PreTrainedModel
+from transformers.utils import logging
+from transformers.modeling_outputs import BaseModelOutput
+
+logger = logging.get_logger(__name__)
+
+from .config import BirdMAEConfig
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (M,)
+    out: (M, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float32)
+    omega /= embed_dim / 2.
+    omega = 1. / 10000**omega  # (D/2,)
+
+    pos = pos.reshape(-1)  # (M,)
+    out = np.einsum('m,d->md', pos, omega)  # (M, D/2), outer product
+
+    emb_sin = np.sin(out) # (M, D/2)
+    emb_cos = np.cos(out) # (M, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
+    return emb
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    # use half of dimensions to encode grid_h
+    emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0])  # (H*W, D/2)
+    emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1])  # (H*W, D/2)
+
+    emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
+    return emb
+
+def get_2d_sincos_pos_embed_flexible(embed_dim, grid_size, cls_token=False):
+    """
+    grid_size: int of the grid height and width
+    return:
+    pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+    """
+    grid_h = np.arange(grid_size[0], dtype=np.float32) # grid size[0] = 8
+    grid_w = np.arange(grid_size[1], dtype=np.float32) # grid size[1] = 32
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0) # 2,8,32
+
+    grid = grid.reshape([2, 1, grid_size[0], grid_size[1]]) # 2,1,8.32
+    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+    if cls_token:
+        pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
+    return pos_embed # 267 (+cls) x 1024 (feature dim)
+
+# From timm.models.weight_init
+def _trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1. + math.erf(x / math.sqrt(2.))) / 2.
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        logging.warning("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+                         f"The distribution may be severely truncated. (Current mean: {mean}, std: {std}, [a, b]: [{a}, {b}])")
+
+    # Values are generated by using a truncated uniform distribution and
+    # then using the inverse CDF for the normal distribution.
+    # Get upper and lower cdf values
+    l = norm_cdf((a - mean) / std)
+    u = norm_cdf((b - mean) / std)
+
+    # Uniformly fill tensor with values from [l, u], then translate to
+    # [2l-1, 2u-1].
+    tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+    # Use inverse cdf transform for normal distribution to get truncated
+    # standard normal
+    tensor.erfinv_()
+
+    # Transform to proper mean, std
+    tensor.mul_(std * math.sqrt(2.))
+    tensor.add_(mean)
+
+    # Clamp to ensure it's in the proper range
+    tensor.clamp_(min=a, max=b)
+    return tensor
+
+def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.):
+    # type: (Tensor, float, float, float, float) -> Tensor
+    """Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are within :math:`[a, b]` interval.
+
+    Args:
+        tensor: an n-dimensional `torch.Tensor`
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+        a: the minimum cutoff value
+        b: the maximum cutoff value
+    """
+    with torch.no_grad():
+        return _trunc_normal_(tensor, mean, std, a, b)
+
+
+# From timm.models.layers
+import collections
+from itertools import repeat
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample  (when applied in main path of residual blocks).
+    """
+    def __init__(self, drop_prob: float = 0., scale_by_keep: bool = True):
+        super(DropPath, self).__init__()
+        self.drop_prob = drop_prob
+        self.scale_by_keep = scale_by_keep
+
+    def forward(self, x):
+        if self.drop_prob == 0. or not self.training:
+            return x
+        keep_prob = 1 - self.drop_prob
+        shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+        random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+        if keep_prob > 0.0 and self.scale_by_keep:
+            random_tensor.div_(keep_prob)
+        return x * random_tensor
+
+def _ntuple(n):
+    def parse(x):
+        if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):
+            return tuple(x)
+        return tuple(repeat(x, n))
+    return parse
+
+class Mlp(nn.Module):
+    """ MLP as used in Vision Transformer, MLP-Mixer and related networks
+    """
+    def __init__(
+            self,
+            in_features,
+            hidden_features=None,
+            out_features=None,
+            act_layer=nn.GELU,
+            norm_layer=None,
+            bias=True,
+            drop=0.,
+            use_conv=False,
+    ):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        bias = _ntuple(2)(bias)
+        drop_probs = _ntuple(2)(drop)
+        linear_layer = partial(nn.Conv2d, kernel_size=1) if use_conv else nn.Linear
+
+        self.fc1 = linear_layer(in_features, hidden_features, bias=bias[0])
+        self.act = act_layer()
+        self.drop1 = nn.Dropout(drop_probs[0])
+        self.norm = norm_layer(hidden_features) if norm_layer is not None else nn.Identity()
+        self.fc2 = linear_layer(hidden_features, out_features, bias=bias[1])
+        self.drop2 = nn.Dropout(drop_probs[1])
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop1(x)
+        x = self.norm(x)
+        x = self.fc2(x)
+        x = self.drop2(x)
+        return x
+
+
+# From timm.models.vision_transformer
+import torch.nn.functional as F
+class Attention(nn.Module):
+    fused_attn: bool
+
+    def __init__(
+            self,
+            dim: int,
+            num_heads: int = 8,
+            qkv_bias: bool = False,
+            qk_norm: bool = False,
+            attn_drop: float = 0.,
+            proj_drop: float = 0.,
+            norm_layer: nn.Module = nn.LayerNorm,
+    ) -> None:
+        super().__init__()
+        assert dim % num_heads == 0, 'dim should be divisible by num_heads'
+        self.num_heads = num_heads
+        self.head_dim = dim // num_heads
+        self.scale = self.head_dim ** -0.5
+        # self.fused_attn = use_fused_attn()
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.q_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
+        self.k_norm = norm_layer(self.head_dim) if qk_norm else nn.Identity()
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)
+        q, k, v = qkv.unbind(0)
+        q, k = self.q_norm(q), self.k_norm(k)
+
+        # if self.fused_attn:
+        x = F.scaled_dot_product_attention(
+            q, k, v,
+            dropout_p=self.attn_drop.p if self.training else 0.,
+        )
+        # else:
+        #     q = q * self.scale
+        #     attn = q @ k.transpose(-2, -1)
+        #     attn = attn.softmax(dim=-1)
+        #     attn = self.attn_drop(attn)
+        #     x = attn @ v
+
+        x = x.transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+# From timm.models.vision_transformer
+class Block(nn.Module):
+    def __init__(
+            self,
+            dim: int,
+            num_heads: int,
+            mlp_ratio: float = 4.,
+            qkv_bias: bool = False,
+            qk_norm: bool = False,
+            proj_drop: float = 0.,
+            attn_drop: float = 0.,
+            init_values: float = None,
+            drop_path: float = 0.,
+            act_layer: nn.Module = nn.GELU,
+            norm_layer: nn.Module = nn.LayerNorm,
+            mlp_layer: nn.Module = Mlp,
+    ) -> None:
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            qk_norm=qk_norm,
+            attn_drop=attn_drop,
+            proj_drop=proj_drop,
+            norm_layer=norm_layer,
+        )
+        self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+
+        self.norm2 = norm_layer(dim)
+        self.mlp = mlp_layer(
+            in_features=dim,
+            hidden_features=int(dim * mlp_ratio),
+            act_layer=act_layer,
+            drop=proj_drop,
+        )
+        self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = x + self.drop_path1(self.ls1(self.attn(self.norm1(x))))
+        x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x))))
+        return x
+
+# From timm.models.vision_transformer
+class LayerScale(nn.Module):
+    def __init__(
+            self,
+            dim: int,
+            init_values: float = 1e-5,
+            inplace: bool = False,
+    ) -> None:
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma
+
+
+
+class PatchEmbed_org(nn.Module):
+    """ Image to Patch Embedding
+    """
+    def __init__(self,
+                 img_size: int | tuple[int, ...] = 224,
+                 patch_size: int | tuple[int, ...] = 16,
+                 in_chans=3,
+                 embed_dim=768):
+        super().__init__()
+        img_size: tuple[int,int] = _ntuple(2)(img_size) # audio mae used: (target_length x 128) --> not sure why tbh
+        patch_size: tuple[int,int] = _ntuple(2)(patch_size)
+        num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0])
+        self.patch_hw = (img_size[1] // patch_size[1], img_size[0] // patch_size[0]) # number of patches height/width = 8/32
+        self.img_size = img_size
+        self.patch_size = patch_size
+        self.num_patches = num_patches
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)
+
+    def forward(self, x):
+        B, C, H, W = x.shape #batch size, channels, height, width --> apparently sth else is expected???
+       # x = x.permute(0,1,3,2) ###????
+        x = self.proj(x) # 1, 1, 512, 128 -> 1, 768, 32, 8 (batch, 768 channel, 32 height, 8 width)
+        x = x.flatten(2) # 1, 768, 32, 8 -> 1, 768, 256
+        x = x.transpose(1, 2) # 1, 768, 256 -> 1, 256, 768
+        return x
+
+
+# --- END OF NECESSARY TIMM/Custom internal module definitions ---
+from functools import partial
+
+class BirdMAEModel(PreTrainedModel):
+    config_class = BirdMAEConfig
+    base_model_prefix = "BirdMAE"
+    main_input_name = "input_values"
+    _auto_class = "AutoModel"
+    _keys_to_ignore_on_load_missing = ["fc_norm.weight", "fc_norm.bias"]
+
+    def __init__(self, config: BirdMAEConfig, **kwargs):
+        super().__init__(config)
+        self.config = config
+
+        # The norm_layer partial is defined within your original MAE_Encoder
+        norm_layer = partial(nn.LayerNorm, eps=config.norm_layer_eps)  # Assuming 1e-6 as default
+
+        self.patch_embed = PatchEmbed_org(
+            img_size=(config.img_size_x, config.img_size_y),  # (512, 128)
+            patch_size=config.patch_size,
+            in_chans=config.in_chans,
+            embed_dim=config.embed_dim
+        )
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, config.embed_dim))
+        self.pos_embed = nn.Parameter(
+            torch.zeros(1, config.num_patches + 1, config.embed_dim),
+            requires_grad=config.pos_trainable
+        )
+
+        # Positional embedding initialization
+        if self.pos_embed.data.shape[1] == config.num_patches + 1:
+            pos_embed_np = get_2d_sincos_pos_embed_flexible(
+                self.pos_embed.shape[-1],  # embedding dim
+                self.patch_embed.patch_hw,  # (8, 32) for a 128x512 image with 16x16 patches
+                cls_token=True
+            )
+            self.pos_embed.data.copy_(torch.from_numpy(pos_embed_np).float().unsqueeze(0))
+        else:
+            logger.warning("Positional embedding shape mismatch. Will not initialize sin-cos pos embed.")
+
+        dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.depth)]
+        self.blocks = nn.ModuleList([
+            Block(
+                dim=config.embed_dim,
+                num_heads=config.num_heads,
+                mlp_ratio=config.mlp_ratio,
+                qkv_bias=config.qkv_bias,
+                qk_norm=config.qk_norm,
+                init_values=config.init_values,
+                proj_drop=config.proj_drop_rate,
+                attn_drop=config.attn_drop_rate,
+                drop_path=dpr[i],
+                norm_layer=norm_layer
+            )
+            for i in range(config.depth)
+        ])
+
+        self.pos_drop = nn.Dropout(p=config.pos_drop_rate)
+        self.norm = norm_layer(config.embed_dim)
+        self.fc_norm = norm_layer(config.embed_dim)
+        self.global_pool = kwargs.get("global_pool", "average")
+
+        trunc_normal_(self.cls_token, std=.02)  # timm uses trunc_normal_
+
+    # used when model is initilized from scratch
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+        elif isinstance(m, nn.Conv2d):  # From your original init_weights
+            w = m.weight.data
+            torch.nn.init.xavier_uniform_(w.view([w.shape[0], -1]))
+
+    def forward(
+            self,
+            input_features: torch.Tensor,  # This will be our spectrograms (B, C, H, W) -> (B, 1, 128, 512)
+            #attention_mask=None,  # For padding (B, num_time_patches)
+            output_attentions: bool = False,
+            output_hidden_states: bool = None,
+            return_dict: bool = None,
+    ):
+        if len(input_features.shape) == 3:
+            input_features = input_features.unsqueeze(0)
+
+        output_attentions = output_attentions or self.config.output_attentions
+
+        if output_attentions:
+            NotImplementedError("output_attention is not yet supported")
+        output_hidden_states = output_hidden_states or self.config.output_hidden_states
+        return_dict = return_dict or self.config.use_return_dict
+
+        B, C, X, Y = input_features.shape
+        assert X == self.config.img_size_x, f"Expected image_size_x={self.config.img_size_x} but was {X}."
+        assert Y == self.config.img_size_y, f"Expected image_size_y={self.config.img_size_y} but was {Y}."
+
+        x = self.patch_embed(input_features)  # Output: (B, num_patches, embed_dim) -> (B, 256, 768)
+
+        x = x + self.pos_embed[:, 1:, :]
+        cls_token = self.cls_token + self.pos_embed[:, :1, :]
+        cls_tokens = cls_token.expand(B, -1, -1)
+        x = torch.cat((cls_tokens, x), dim=1)
+        x = self.pos_drop(x)
+
+        all_hidden_states = (x,) if output_hidden_states else None
+        for blk in self.blocks:
+            x = blk(x)
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (x,)
+
+        if self.global_pool == "average":
+            x = x[:, 1:, :].mean(dim=1)
+            pooled_output = self.fc_norm(x)
+        elif self.global_pool == "cls":
+            x = self.norm(x)
+            pooled_output = x[:, 0]
+        else:
+            raise ValueError(f"Invalid global pool type: {self.global_pool}")
+
+        if not return_dict:
+            return (pooled_output,) + (all_hidden_states if output_hidden_states else ()) + (None,)
+
+        return BaseModelOutput(
+            last_hidden_state=pooled_output,
+            hidden_states=all_hidden_states,
+            attentions=None
+        )

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:402103ecc81787ca6317ad78377fb38a912b3548f23a18f424d7a0dc31f754c5
+size 341826344

preprocessor_config.json

@@ -0,0 +1,20 @@
+{
+  "auto_map": {
+    "AutoFeatureExtractor": "feature_extractor.BirdMAEFeatureExtractor"
+  },
+  "dither": 0.0,
+  "feature_extractor_type": "BirdMAEFeatureExtractor",
+  "feature_size": 1,
+  "frame_shift": 10,
+  "htk_compat": true,
+  "mean": -7.2,
+  "num_mel_bins": 128,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 32000,
+  "std": 4.43,
+  "target_length": 512,
+  "use_energy": false,
+  "window_type": "hanning"
+}