Model inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d551237b-1866-4a8b-a51f-a736464893ee",
   "metadata": {},
   "source": [
    "# Audio Gender Classification Pipeline\n",
    "\n",
    "*This is the third notebook out of three notebooks focused on Model Inference.*\n",
    "\n",
    "## 1. Audio Recording\n",
    "- **Step:** Record audio directly from the microphone.\n",
    "- **Details:** The function records audio for a specified duration (in seconds) and saves the recorded audio as a `.wav` file.\n",
    "\n",
    "## 2. Voice Activity Detection (VAD)\n",
    "- **Step:** Detect if there's any speech in the recorded audio using the Silero VAD model.\n",
    "- **Details:** \n",
    "  - The function loads the `.wav` file.\n",
    "  - Converts stereo audio to mono if necessary.\n",
    "  - Detects speech segments and returns speech timestamps if any speech is detected.\n",
    "\n",
    "## 3. Audio Preprocessing\n",
    "- **Step:** Preprocess the audio for gender classification.\n",
    "- **Details:** \n",
    "  - The audio is resampled to 16kHz if necessary.\n",
    "  - Stereo audio is converted to mono.\n",
    "  - The audio is prepared as input for the Wav2Vec2 model by tokenizing it.\n",
    "\n",
    "## 4. Gender Prediction\n",
    "- **Step:** Predict the gender of the speaker using a fine-tuned Wav2Vec2 model.\n",
    "- **Details:** \n",
    "  - The model processes the input audio and outputs logits.\n",
    "  - The predicted label (Male or Female) is determined by analyzing the logits.\n",
    "\n",
    "## 5. Pipeline Execution\n",
    "- **Step:** Run the entire pipeline from audio recording to gender prediction.\n",
    "- **Details:** \n",
    "  - The function first detects voice activity in the audio.\n",
    "  - If speech is detected, the gender of the speaker is predicted.\n",
    "  - The predicted gender is displayed.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2e9d9a07-d289-44ed-ab3c-1b838da25b64",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sounddevice as sd\n",
    "import numpy as np\n",
    "import noisereduce as nr\n",
    "import scipy.io.wavfile as wavfile\n",
    "import torchaudio\n",
    "import torch\n",
    "from transformers import Wav2Vec2Processor, Wav2Vec2ForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8518f703-0d61-4bec-9291-5f1135cfd681",
   "metadata": {},
   "outputs": [],
   "source": [
    "def record_audio(duration=6, sample_rate=16000, output_file=\"recorded_audio.wav\"):\n",
    "    print(f\"Recording for {duration} seconds...\")\n",
    "    audio_data = sd.rec(int(duration * sample_rate), samplerate=sample_rate, channels=1, dtype=np.float32)\n",
    "    sd.wait()  # Wait until the recording is finished\n",
    "    wavfile.write(output_file, sample_rate, audio_data)\n",
    "    print(f\"Recording saved as {output_file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b8984cc5-3d7b-4fca-9bdd-b49c8ee28dc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_wav_file(file_path):\n",
    "    wav, sr = torchaudio.load(file_path)\n",
    "    return wav, sr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fe171db1-8d20-4fbe-988d-7391890be68d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://github.com/snakers4/silero-vad/zipball/master\" to C:\\Users\\Mostafa/.cache\\torch\\hub\\master.zip\n"
     ]
    }
   ],
   "source": [
    "model_vad, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', \n",
    "                                  model='silero_vad', \n",
    "                                  force_reload=True, \n",
    "                                  trust_repo=True)\n",
    "\n",
    "# Extract utilities from Silero VAD\n",
    "(get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks) = utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a2a292ff-9698-44fa-862b-838f3a536607",
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_voice_activity(wav_file_path):\n",
    "    wav, sr = load_wav_file(wav_file_path)\n",
    "    \n",
    "    # Convert stereo to mono \n",
    "    if wav.shape[0] > 1:\n",
    "        wav = torch.mean(wav, dim=0, keepdim=True)\n",
    "    \n",
    "    wav_np = wav.numpy()[0]\n",
    "\n",
    "    wav_np_denoised = nr.reduce_noise(y=wav_np, sr=sr)\n",
    "\n",
    "    wav_np_denoised = wav_np_denoised / np.max(np.abs(wav_np_denoised))  # Normalize to [-1, 1]\n",
    "    \n",
    "    wav = torch.tensor(wav_np_denoised, dtype=torch.float32).unsqueeze(0)  # Add channel dimension\n",
    "    \n",
    "    speech_timestamps = get_speech_timestamps(wav.squeeze(), model_vad, sampling_rate=sr)\n",
    "    \n",
    "    if speech_timestamps:\n",
    "        print(\"Voice detected in the audio.\")\n",
    "    else:\n",
    "        print(\"No voice detected in the audio.\")\n",
    "    \n",
    "    return speech_timestamps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bfc6e474-2bc7-4b23-b064-6772695c2802",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_gender(audio_path):\n",
    "    speech_array, sampling_rate = torchaudio.load(audio_path)\n",
    "    \n",
    "    if sampling_rate != 16000:  # assuming Wav2Vec2 expects 16kHz\n",
    "        resampler = torchaudio.transforms.Resample(sampling_rate, 16000)\n",
    "        speech_array = resampler(speech_array)\n",
    "    \n",
    "    # Convert stereo to mono if needed (Wav2Vec2 expects single channel audio)\n",
    "    if speech_array.shape[0] > 1:\n",
    "        speech_array = torch.mean(speech_array, dim=0, keepdim=True)\n",
    "    \n",
    "    # Remove the extra dimension from speech_array (ensure it has shape [1, sequence_length])\n",
    "    speech_array = speech_array.squeeze()  \n",
    "    \n",
    "    # Ensure the input shape is [batch_size, sequence_length]\n",
    "    inputs = processor(speech_array, sampling_rate=16000, return_tensors=\"pt\", padding=True)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        logits = model(**inputs).logits\n",
    "        predicted_label = torch.argmax(logits, dim=-1).item()\n",
    "\n",
    "    return \"Male\" if predicted_label == 0 else \"Female\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "84d25643-a6ce-4b66-a0f6-3aaae766782d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main_pipeline(audio_path):\n",
    "    # Step 1: Detect voice activity using Silero VAD\n",
    "    speech_timestamps = detect_voice_activity(audio_path)\n",
    "    \n",
    "    # Step 2: If voice is detected, predict the gender\n",
    "    if speech_timestamps:\n",
    "        predicted_gender = predict_gender(audio_path)\n",
    "        print(f\"Predicted Gender: {predicted_gender}\")\n",
    "    else:\n",
    "        print(\"Skipping gender prediction due to no voice detected.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0d0001b6-b47c-4c10-9402-9e881c1af9f2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recording for 6 seconds...\n",
      "Recording saved as recorded_audio.wav\n",
      "Voice detected in the audio.\n",
      "WARNING:tensorflow:From C:\\Users\\Mostafa\\anaconda3\\Lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n",
      "Predicted Gender: Male\n"
     ]
    }
   ],
   "source": [
    "# Load the saved Wav2Vec2 gender classification model and processor\n",
    "model_dir = \"model_output\"\n",
    "model = Wav2Vec2ForSequenceClassification.from_pretrained(model_dir)\n",
    "processor = Wav2Vec2Processor.from_pretrained(model_dir)\n",
    "\n",
    "record_audio(duration=6, sample_rate=16000, output_file=\"recorded_audio.wav\")\n",
    "\n",
    "audio_path = \"recorded_audio.wav\"\n",
    "main_pipeline(audio_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37ca8394-b41d-4ba4-874b-a1f7003a3aa4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}