Wav2Vec2_0_HuggingFace.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Wav2Vec2.0_HuggingFace",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "ChWFBCwZ3ysw"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GmrMN9BDBJRR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c56031b-555e-42fc-9dfa-7218f0434d37"
      },
      "source": [
        "# Download sample audio file\n",
        "!gdown --id 1GkDBSWODEIxYijPua_hL4jRP3iquYFIA"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1GkDBSWODEIxYijPua_hL4jRP3iquYFIA\n",
            "To: /content/p225_003.wav\n",
            "\r  0% 0.00/192k [00:00<?, ?B/s]\r100% 192k/192k [00:00<00:00, 29.4MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_4w4rlk1_BHa"
      },
      "source": [
        "## Wav2Vec2.0 for Hidden Representations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lIYdn1woOS1n",
        "outputId": "cfde9f5b-b6a8-40c5-ce35-22af51f9becf"
      },
      "source": [
        "import soundfile as sf\n",
        "import torch\n",
        "from transformers import Wav2Vec2Model, Wav2Vec2Tokenizer, Wav2Vec2ForMaskedLM, Wav2Vec2ForCTC\n",
        "\n",
        "# load pretrained model\n",
        "tokenizer = Wav2Vec2Tokenizer.from_pretrained(\"facebook/wav2vec2-base-960h\")\n",
        "model = Wav2Vec2Model.from_pretrained(\"facebook/wav2vec2-base-960h\")\n",
        "\n",
        "# load audio\n",
        "# audio_input, _ = sf.read(\"path/to/audio/file\")\n",
        "\n",
        "audio_input, _ = sf.read(\"/content/p225_003.wav\")\n",
        "\n",
        "\n",
        "# Applying tokenizer\n",
        "input_values = tokenizer(audio_input, return_tensors=\"pt\").input_values \n",
        "\n",
        "# Run Model\n",
        "hidden_state = model(input_values).last_hidden_state\n",
        "\n",
        "print(hidden_state.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 300, 768])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TVFx0ISK_bls"
      },
      "source": [
        "**Note**: InOrder to train any new model with lesser amount of training data we can use these hidden representations and directly feed into our network instead of using feature extractors like mfcc, filter banks.."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RD-8pTj3_6O1"
      },
      "source": [
        "## Wa2Vec2.0 with MaskedLM Head"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1gc9Nv7J-0IQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2ded7686-a5de-4c89-d4e3-9fb50ebc4e3d"
      },
      "source": [
        "# load audio\n",
        "# audio_input, _ = sf.read(\"path/to/audio/file\")\n",
        "\n",
        "model = Wav2Vec2ForMaskedLM.from_pretrained(\"facebook/wav2vec2-base-960h\")\n",
        "audio_input, _ = sf.read(\"/content/p225_003.wav\")\n",
        "\n",
        "# transcribe\n",
        "input_values = tokenizer(audio_input, return_tensors=\"pt\").input_values\n",
        "logits = model(input_values).logits\n",
        "predicted_ids = torch.argmax(logits, dim=-1)\n",
        "transcription = tokenizer.batch_decode(predicted_ids)[0]\n",
        "\n",
        "print(transcription)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/models/wav2vec2/modeling_wav2vec2.py:670: FutureWarning: The class `Wav2Vec2ForMaskedLM` is deprecated. Please use `Wav2Vec2ForCTC` instead.\n",
            "  \"The class `Wav2Vec2ForMaskedLM` is deprecated. Please use `Wav2Vec2ForCTC` instead.\", FutureWarning\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "SIX SPOONS OF FRESH SNOW PEESE FIVE THIK SLABS OF BLUE CHEESE AND MAY BE A STACK FOR HER BROTHER BOB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3UpMvC4tAezl"
      },
      "source": [
        "## Wav2Vec2.0 with CTC"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ArKfxz5VANix",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4b5fd3ec-6560-4199-e0ca-7b4464847388"
      },
      "source": [
        "model = Wav2Vec2ForCTC.from_pretrained(\"facebook/wav2vec2-base-960h\")\n",
        "\n",
        "\n",
        "input_values = tokenizer(audio_input, return_tensors=\"pt\").input_values  # Batch size 1\n",
        "logits = model(input_values).logits\n",
        "\n",
        "predicted_ids = torch.argmax(logits, dim=-1)\n",
        "transcription = tokenizer.decode(predicted_ids[0])\n",
        "\n",
        "print(transcription)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SIX SPOONS OF FRESH SNOW PEESE FIVE THIK SLABS OF BLUE CHEESE AND MAY BE A STACK FOR HER BROTHER BOB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aL6Uv7XfA1nZ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}