VirgoolInformal_Dataset_Processing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "3AcYoP960ry-"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Environment Setup"
      ],
      "metadata": {
        "id": "qm2b1llYZMlY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install hazm  # Requires Restart"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 879
        },
        "id": "sUOKF5SYZhgW",
        "outputId": "12400790-d49e-40a6-c797-6492aad77f2a"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting hazm\n",
            "  Downloading hazm-0.10.0-py3-none-any.whl (892 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m892.6/892.6 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fasttext-wheel<0.10.0,>=0.9.2 (from hazm)\n",
            "  Downloading fasttext_wheel-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.4/4.4 MB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting flashtext<3.0,>=2.7 (from hazm)\n",
            "  Downloading flashtext-2.7.tar.gz (14 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: gensim<5.0.0,>=4.3.1 in /usr/local/lib/python3.10/dist-packages (from hazm) (4.3.2)\n",
            "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /usr/local/lib/python3.10/dist-packages (from hazm) (3.8.1)\n",
            "Collecting numpy==1.24.3 (from hazm)\n",
            "  Downloading numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.3/17.3 MB\u001b[0m \u001b[31m21.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-crfsuite<0.10.0,>=0.9.9 (from hazm)\n",
            "  Downloading python_crfsuite-0.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scikit-learn<2.0.0,>=1.2.2 in /usr/local/lib/python3.10/dist-packages (from hazm) (1.2.2)\n",
            "Collecting pybind11>=2.2 (from fasttext-wheel<0.10.0,>=0.9.2->hazm)\n",
            "  Downloading pybind11-2.12.0-py3-none-any.whl (234 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m235.0/235.0 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: setuptools>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from fasttext-wheel<0.10.0,>=0.9.2->hazm) (67.7.2)\n",
            "Requirement already satisfied: scipy>=1.7.0 in /usr/local/lib/python3.10/dist-packages (from gensim<5.0.0,>=4.3.1->hazm) (1.11.4)\n",
            "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.10/dist-packages (from gensim<5.0.0,>=4.3.1->hazm) (6.4.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (2024.5.15)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk<4.0.0,>=3.8.1->hazm) (4.66.4)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn<2.0.0,>=1.2.2->hazm) (3.5.0)\n",
            "Building wheels for collected packages: flashtext\n",
            "  Building wheel for flashtext (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for flashtext: filename=flashtext-2.7-py2.py3-none-any.whl size=9296 sha256=1c7ed8269d7ca3c0838a069be159d8aee981e9ed58934caebf5c66db1193f70d\n",
            "  Stored in directory: /root/.cache/pip/wheels/bc/be/39/c37ad168eb2ff644c9685f52554440372129450f0b8ed203dd\n",
            "Successfully built flashtext\n",
            "Installing collected packages: python-crfsuite, flashtext, pybind11, numpy, fasttext-wheel, hazm\n",
            "  Attempting uninstall: numpy\n",
            "    Found existing installation: numpy 1.25.2\n",
            "    Uninstalling numpy-1.25.2:\n",
            "      Successfully uninstalled numpy-1.25.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "pandas-stubs 2.0.3.230814 requires numpy>=1.25.0; python_version >= \"3.9\", but you have numpy 1.24.3 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed fasttext-wheel-0.9.2 flashtext-2.7 hazm-0.10.0 numpy-1.24.3 pybind11-2.12.0 python-crfsuite-0.9.10\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "numpy"
                ]
              },
              "id": "3a39f6c901ec42aa8fcdf3a3439f33ec"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install pydub"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cFSxcaOjW1ij",
        "outputId": "be7f68a5-f4d1-4511-e750-842bb4fdfa69"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pydub\n",
            "  Using cached pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n",
            "Installing collected packages: pydub\n",
            "Successfully installed pydub-0.25.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Setup parsi io"
      ],
      "metadata": {
        "id": "NVlJhS9_cHaJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! git clone https://github.com/language-ml/parsi.io.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lrPGOeS6Zz-b",
        "outputId": "6017ed27-4953-4155-a9c8-be5451d8ae70"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fatal: destination path 'parsi.io' already exists and is not an empty directory.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mv parsi.io parsi_io"
      ],
      "metadata": {
        "id": "eEROzSSgaAUK"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ILGxoNVgwDPy"
      },
      "source": [
        "## Setup Perpos POS Tagger"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4d2fX0Nkx-c_",
        "outputId": "939fbbc7-4994-4818-d80b-e533673c811e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting sklearn-crfsuite\n",
            "  Downloading sklearn_crfsuite-0.3.6-py2.py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: python-crfsuite>=0.8.3 in /usr/local/lib/python3.10/dist-packages (from sklearn-crfsuite) (0.9.10)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from sklearn-crfsuite) (1.16.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from sklearn-crfsuite) (0.9.0)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.10/dist-packages (from sklearn-crfsuite) (4.66.4)\n",
            "Installing collected packages: sklearn-crfsuite\n",
            "Successfully installed sklearn-crfsuite-0.3.6\n"
          ]
        }
      ],
      "source": [
        "! pip install sklearn-crfsuite"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3FC4zlmoxOD2",
        "outputId": "e2094ec6-522d-4932-ffb9-228a7665dab1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'perpos'...\n",
            "remote: Enumerating objects: 41, done.\u001b[K\n",
            "remote: Total 41 (delta 0), reused 0 (delta 0), pack-reused 41\u001b[K\n",
            "Receiving objects: 100% (41/41), 17.33 MiB | 8.56 MiB/s, done.\n",
            "Resolving deltas: 100% (11/11), done.\n"
          ]
        }
      ],
      "source": [
        "! git clone https://github.com/mhbashari/perpos.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "1wT7lcQLxTkd"
      },
      "outputs": [],
      "source": [
        "import string\n",
        "\n",
        "from nltk import tree2conlltags\n",
        "\n",
        "\n",
        "def read_conll(path, col=2):\n",
        "    with open(path, \"r\", encoding=\"utf-8\") as conll:\n",
        "        out = []\n",
        "        for sent in conll.readlines():\n",
        "            split = sent.strip(\"\\r\\n\").split()\n",
        "            if len(split) > 1:\n",
        "                none_token_count = col - 1\n",
        "                new_elem = split[-1:]\n",
        "                new_elem = split[:none_token_count] + new_elem\n",
        "                out.append(new_elem)\n",
        "\n",
        "            else:\n",
        "                yield out\n",
        "                out = []\n",
        "\n",
        "\n",
        "def template(word):\n",
        "    return \"\".join([(lambda item: \"x\" if not item in \"آایو\" else \"a\")(char) for char in word])\n",
        "\n",
        "\n",
        "def isdigit(word):\n",
        "    return all(map(lambda char: char in \"۱۲۳۴۵۶۷۸۹۰1234567890.\", word))\n",
        "\n",
        "\n",
        "def ngram(word, leng=2):\n",
        "    for i in range(len(word) - 1):\n",
        "        yield 'word[' + str(i) + \":\" + str(i + leng) + \"]\", word[i:i + leng]\n",
        "\n",
        "\n",
        "def tree2brackets(tree):\n",
        "    str, tag = '', ''\n",
        "    for item in tree2conlltags(tree):\n",
        "        if item[2][0] in {'B', 'O'} and tag:\n",
        "            str += tag + '] '\n",
        "            tag = ''\n",
        "\n",
        "        if item[2][0] == 'B':\n",
        "            tag = item[2].split('-')[1]\n",
        "            str += '['\n",
        "        str += item[0] + ' '\n",
        "\n",
        "    if tag:\n",
        "        str += tag + '] '\n",
        "\n",
        "    return str.strip()\n",
        "\n",
        "def word2features(sent, i):\n",
        "    W = sent[i]\n",
        "    features = {\n",
        "        'B': 1.0,\n",
        "        'W': W,\n",
        "        'P': W in string.punctuation,\n",
        "        'T': template(W),\n",
        "        'D(W)': isdigit(W),\n",
        "    }\n",
        "    for leng in range(max(4 + 1, len(W)) + 1):\n",
        "        for k, v in ngram(W, leng=leng):\n",
        "            features[k] = v\n",
        "    if i > 0:\n",
        "        W = sent[i - 1][0]\n",
        "        features.update({\n",
        "            '-1W[-3': W[-3:],\n",
        "            '-1W[-2': W[-2:],\n",
        "            '-1W[-1': W[-1:],\n",
        "            '-1W': W,\n",
        "            '-1W0W': W + sent[i],\n",
        "            '-1P': W in string.punctuation,\n",
        "            '-1T': template(W)\n",
        "        })\n",
        "    else:\n",
        "        features['BOS'] = True\n",
        "    if i > 1:\n",
        "        W = sent[i - 2][0]\n",
        "        features.update({\n",
        "            '-2W[-3': W[-3:],\n",
        "            '-2W[-2': W[-2:],\n",
        "            '-2W[-1': W[-1:],\n",
        "            '-2P': W in string.punctuation,\n",
        "            '-2T': template(W)\n",
        "        })\n",
        "\n",
        "    if i < len(sent) - 2:\n",
        "        W = sent[i + 2][0]\n",
        "        features.update({\n",
        "            '+2W[-1': W[-1:],\n",
        "            '+2W[-2': W[-2:],\n",
        "            '+2W': W,\n",
        "            '+2P': W in string.punctuation,\n",
        "            '+2T': template(W)\n",
        "        })\n",
        "    if i < len(sent) - 1:\n",
        "        W = sent[i + 1][0]\n",
        "        features.update({\n",
        "            '+1W[-1': W[-1:],\n",
        "            '+1W': W,\n",
        "            '+1W0W': W + sent[i],\n",
        "            '+1W[-2': W[-2:],\n",
        "            '+1:P': W in string.punctuation,\n",
        "            '+1:T': template(W)\n",
        "        })\n",
        "    else:\n",
        "        features['EOS'] = True\n",
        "    if 0 < i < len(sent) - 1:\n",
        "        features['-1W/+1W'] = sent[i + 1][0] + \"/\" + sent[i - 1][0]\n",
        "    return features\n",
        "\n",
        "\n",
        "def token2features(token_list):\n",
        "    return [word2features(token_list, i) for i in range(len(token_list))]\n",
        "\n",
        "\n",
        "def sent2labels(sent):\n",
        "    return [postag for token, postag in sent]\n",
        "\n",
        "\n",
        "def sent2tokens(sent):\n",
        "    return [token for token, postag in sent]\n",
        "\n",
        "\n",
        "import pickle\n",
        "\n",
        "class POSTagger:\n",
        "    def __init__(self, model_path):\n",
        "        self.model_path = model_path\n",
        "        self.crf = pickle.load(open(model_path, \"rb\"))\n",
        "\n",
        "    def parse(self, token_stream):\n",
        "        return self.parse_sentences([token_stream])[0]\n",
        "\n",
        "    def parse_sentences(self, list_of_token_stream):\n",
        "        X_test = [token2features(s) for s in list_of_token_stream]\n",
        "        y_pred = self.crf.predict(X_test)\n",
        "        out = []\n",
        "        for x_sent, y_pred in zip(list_of_token_stream, y_pred):\n",
        "            out.append(list(zip(x_sent, y_pred)))\n",
        "        return out\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "HhkXRbAoxcoe"
      },
      "outputs": [],
      "source": [
        "pos_tagger = POSTagger(\"perpos/model/perpos.model\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3AcYoP960ry-"
      },
      "source": [
        "## Setup Aeneas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wUyP7wCo1J59",
        "outputId": "62623d7a-1e9d-4324-a0e0-2bd60cae8d5b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.8/16.8 MB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "chex 0.1.86 requires numpy>=1.24.1, but you have numpy 1.22.4 which is incompatible.\n",
            "cudf-cu12 24.4.1 requires numpy<2.0a0,>=1.23, but you have numpy 1.22.4 which is incompatible.\n",
            "hazm 0.10.0 requires numpy==1.24.3, but you have numpy 1.22.4 which is incompatible.\n",
            "pandas-stubs 2.0.3.230814 requires numpy>=1.25.0; python_version >= \"3.9\", but you have numpy 1.22.4 which is incompatible.\n",
            "plotnine 0.12.4 requires numpy>=1.23.0, but you have numpy 1.22.4 which is incompatible.\n",
            "rmm-cu12 24.4.0 requires numpy<2.0a0,>=1.23, but you have numpy 1.22.4 which is incompatible.\n",
            "tensorflow 2.15.0 requires numpy<2.0.0,>=1.23.5, but you have numpy 1.22.4 which is incompatible.\u001b[0m\u001b[31m\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "ffmpeg is already the newest version (7:4.4.2-0ubuntu0.22.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 45 not upgraded.\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "The following additional packages will be installed:\n",
            "  espeak-data libespeak1 libportaudio2 libsonic0\n",
            "The following NEW packages will be installed:\n",
            "  espeak espeak-data libespeak1 libportaudio2 libsonic0\n",
            "0 upgraded, 5 newly installed, 0 to remove and 45 not upgraded.\n",
            "Need to get 1,382 kB of archives.\n",
            "After this operation, 3,178 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libportaudio2 amd64 19.6.0-1.1 [65.3 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libsonic0 amd64 0.2.0-11build1 [10.3 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 espeak-data amd64 1.48.15+dfsg-3 [1,085 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libespeak1 amd64 1.48.15+dfsg-3 [156 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy/universe amd64 espeak amd64 1.48.15+dfsg-3 [64.2 kB]\n",
            "Fetched 1,382 kB in 1s (1,302 kB/s)\n",
            "Selecting previously unselected package libportaudio2:amd64.\n",
            "(Reading database ... 121918 files and directories currently installed.)\n",
            "Preparing to unpack .../libportaudio2_19.6.0-1.1_amd64.deb ...\n",
            "Unpacking libportaudio2:amd64 (19.6.0-1.1) ...\n",
            "Selecting previously unselected package libsonic0:amd64.\n",
            "Preparing to unpack .../libsonic0_0.2.0-11build1_amd64.deb ...\n",
            "Unpacking libsonic0:amd64 (0.2.0-11build1) ...\n",
            "Selecting previously unselected package espeak-data:amd64.\n",
            "Preparing to unpack .../espeak-data_1.48.15+dfsg-3_amd64.deb ...\n",
            "Unpacking espeak-data:amd64 (1.48.15+dfsg-3) ...\n",
            "Selecting previously unselected package libespeak1:amd64.\n",
            "Preparing to unpack .../libespeak1_1.48.15+dfsg-3_amd64.deb ...\n",
            "Unpacking libespeak1:amd64 (1.48.15+dfsg-3) ...\n",
            "Selecting previously unselected package espeak.\n",
            "Preparing to unpack .../espeak_1.48.15+dfsg-3_amd64.deb ...\n",
            "Unpacking espeak (1.48.15+dfsg-3) ...\n",
            "Setting up libportaudio2:amd64 (19.6.0-1.1) ...\n",
            "Setting up libsonic0:amd64 (0.2.0-11build1) ...\n",
            "Setting up espeak-data:amd64 (1.48.15+dfsg-3) ...\n",
            "Setting up libespeak1:amd64 (1.48.15+dfsg-3) ...\n",
            "Setting up espeak (1.48.15+dfsg-3) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "libgdal-dev is already the newest version (3.6.4+dfsg-1~jammy0).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 45 not upgraded.\n"
          ]
        }
      ],
      "source": [
        "!pip install -q numpy==1.22.4\n",
        "!apt-get install ffmpeg\n",
        "!apt-get install espeak\n",
        "!pip install -q beautifulsoup4\n",
        "!pip install -q lxml\n",
        "!apt-get install libgdal-dev"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ibxBKfbQ1MAv",
        "outputId": "adbe9c70-6bef-4b7f-b421-e4aecd0c5182"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2024-06-03 15:43:07--  https://raw.githubusercontent.com/readbeyond/aeneas/master/install_dependencies.sh\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 2759 (2.7K) [text/plain]\n",
            "Saving to: ‘install_dependencies.sh’\n",
            "\n",
            "install_dependencie 100%[===================>]   2.69K  --.-KB/s    in 0s      \n",
            "\n",
            "2024-06-03 15:43:07 (32.3 MB/s) - ‘install_dependencies.sh’ saved [2759/2759]\n",
            "\n",
            "[INFO] A.1 Adding deb-multimedia to apt sources...\n",
            "[INFO] A.1 Adding deb-multimedia to apt sources... done\n",
            "[INFO] A.2 Updating apt...\n",
            "Get:1 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB]\n",
            "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Get:3 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease [3,626 B]\n",
            "Hit:4 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB]\n",
            "Ign:6 http://www.deb-multimedia.org jessie InRelease\n",
            "Err:7 http://www.deb-multimedia.org jessie Release\n",
            "  404  Not Found [IP: 188.165.241.192 80]\n",
            "Hit:8 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n",
            "Hit:9 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB]\n",
            "Get:11 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease [24.3 kB]\n",
            "Get:12 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,858 kB]\n",
            "Hit:13 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [1,084 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,377 kB]\n",
            "Get:16 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy/main amd64 Packages [47.6 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,129 kB]\n",
            "Reading package lists... Done\n",
            "E: The repository 'http://www.deb-multimedia.org jessie Release' does not have a Release file.\n",
            "N: Updating from such a repository can't be done securely, and is therefore disabled by default.\n",
            "N: See apt-secure(8) manpage for repository creation and user configuration details.\n",
            "[INFO] A.2 Updating apt... done\n",
            "[INFO] A.3 Downloading and installing deb-multimedia keyring...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "W: --force-yes is deprecated, use one of the options starting with --allow instead.\n",
            "E: Unable to locate package deb-multimedia-keyring\n",
            "[INFO] A.3 Downloading and installing deb-multimedia keyring... done\n",
            "[INFO] A.4 Updating apt...\n",
            "Hit:1 http://security.ubuntu.com/ubuntu jammy-security InRelease\n",
            "Hit:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64  InRelease\n",
            "Hit:3 https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/ InRelease\n",
            "Ign:4 http://www.deb-multimedia.org jessie InRelease\n",
            "Hit:5 http://archive.ubuntu.com/ubuntu jammy InRelease\n",
            "Err:6 http://www.deb-multimedia.org jessie Release\n",
            "  404  Not Found [IP: 188.165.241.192 80]\n",
            "Hit:7 http://archive.ubuntu.com/ubuntu jammy-updates InRelease\n",
            "Hit:8 http://archive.ubuntu.com/ubuntu jammy-backports InRelease\n",
            "Hit:9 https://ppa.launchpadcontent.net/c2d4u.team/c2d4u4.0+/ubuntu jammy InRelease\n",
            "Hit:10 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy InRelease\n",
            "Hit:11 https://ppa.launchpadcontent.net/graphics-drivers/ppa/ubuntu jammy InRelease\n",
            "Hit:12 https://ppa.launchpadcontent.net/ubuntugis/ppa/ubuntu jammy InRelease\n",
            "Reading package lists... Done\n",
            "E: The repository 'http://www.deb-multimedia.org jessie Release' does not have a Release file.\n",
            "N: Updating from such a repository can't be done securely, and is therefore disabled by default.\n",
            "N: See apt-secure(8) manpage for repository creation and user configuration details.\n",
            "[INFO] A.4 Updating apt... done\n",
            "[INFO] B.1 Installing ffmpeg (from deb-multimedia)...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "ffmpeg is already the newest version (7:4.4.2-0ubuntu0.22.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 51 not upgraded.\n",
            "W: --force-yes is deprecated, use one of the options starting with --allow instead.\n",
            "[INFO] B.1 Installing ffmpeg (from deb-multimedia)... done\n",
            "[INFO] B.2 Installing espeak...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "espeak is already the newest version (1.48.15+dfsg-3).\n",
            "espeak-data is already the newest version (1.48.15+dfsg-3).\n",
            "espeak-data set to manually installed.\n",
            "libespeak1 is already the newest version (1.48.15+dfsg-3).\n",
            "libespeak1 set to manually installed.\n",
            "The following NEW packages will be installed:\n",
            "  libespeak-dev\n",
            "0 upgraded, 1 newly installed, 0 to remove and 51 not upgraded.\n",
            "Need to get 193 kB of archives.\n",
            "After this operation, 624 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libespeak-dev amd64 1.48.15+dfsg-3 [193 kB]\n",
            "Fetched 193 kB in 1s (324 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 1.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libespeak-dev:amd64.\n",
            "(Reading database ... 122248 files and directories currently installed.)\n",
            "Preparing to unpack .../libespeak-dev_1.48.15+dfsg-3_amd64.deb ...\n",
            "Unpacking libespeak-dev:amd64 (1.48.15+dfsg-3) ...\n",
            "Setting up libespeak-dev:amd64 (1.48.15+dfsg-3) ...\n",
            "[INFO] B.2 Installing espeak... done\n",
            "[INFO] B.3 Installing festival...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Note, selecting 'festival-voice' for glob 'festival*'\n",
            "Note, selecting 'festival-catalan-voice' for glob 'festival*'\n",
            "Note, selecting 'festival-czech' for glob 'festival*'\n",
            "Note, selecting 'festival-ca' for glob 'festival*'\n",
            "Note, selecting 'festival-hi' for glob 'festival*'\n",
            "Note, selecting 'festival-mr' for glob 'festival*'\n",
            "Note, selecting 'festival-te' for glob 'festival*'\n",
            "Note, selecting 'festival-freebsoft-utils' for glob 'festival*'\n",
            "Note, selecting 'festival-dev' for glob 'festival*'\n",
            "Note, selecting 'festival-doc' for glob 'festival*'\n",
            "Note, selecting 'festival' for glob 'festival*'\n",
            "Note, selecting 'festvox-ca-ona-hts' instead of 'festival-catalan-voice'\n",
            "The following additional packages will be installed:\n",
            "  alsa-utils doc-base festlex-cmu festlex-poslex festvox-czech-ph\n",
            "  festvox-hi-nsk festvox-kallpc16k festvox-mr-nsk festvox-te-nsk libatopology2\n",
            "  libestools-dev libestools2.5 libfftw3-single3 libopencore-amrnb0\n",
            "  libopencore-amrwb0 libsox-fmt-alsa libsox-fmt-base libsox3 libuuid-perl\n",
            "  libwavpack1 libyaml-tiny-perl sgml-base sox\n",
            "Suggested packages:\n",
            "  dialog dhelp | dwww | dochelp | doc-central | yelp | khelpcenter\n",
            "  pidgin-festival speech-tools-doc libfftw3-bin libfftw3-dev libsox-fmt-all\n",
            "  sgml-base-doc\n",
            "The following NEW packages will be installed:\n",
            "  alsa-utils doc-base festival festival-ca festival-czech festival-dev\n",
            "  festival-doc festival-freebsoft-utils festival-hi festival-mr festival-te\n",
            "  festlex-cmu festlex-poslex festvox-ca-ona-hts festvox-czech-ph\n",
            "  festvox-hi-nsk festvox-kallpc16k festvox-mr-nsk festvox-te-nsk libatopology2\n",
            "  libestools-dev libestools2.5 libfftw3-single3 libopencore-amrnb0\n",
            "  libopencore-amrwb0 libsox-fmt-alsa libsox-fmt-base libsox3 libuuid-perl\n",
            "  libwavpack1 libyaml-tiny-perl sgml-base sox\n",
            "0 upgraded, 33 newly installed, 0 to remove and 51 not upgraded.\n",
            "Need to get 50.5 MB of archives.\n",
            "After this operation, 139 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 sgml-base all 1.30 [12.5 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libatopology2 amd64 1.2.6.1-1ubuntu1 [51.3 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 libfftw3-single3 amd64 3.3.8-2ubuntu8 [800 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 alsa-utils amd64 1.2.6-1ubuntu1 [1,177 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu jammy/main amd64 libuuid-perl amd64 0.28-1build4 [15.9 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu jammy/main amd64 libyaml-tiny-perl all 1.73-1 [25.2 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu jammy/main amd64 doc-base all 0.11.1 [79.4 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libestools2.5 amd64 1:2.5.0-12 [1,038 kB]\n",
            "Get:9 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival amd64 1:2.5.0-8 [751 kB]\n",
            "Get:10 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-ca all 3.0.6-2 [2,695 kB]\n",
            "Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-czech all 0.3-5 [31.5 kB]\n",
            "Get:12 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-doc all 1:2.5.0-8 [648 kB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-freebsoft-utils all 0.10-9 [48.0 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festvox-hi-nsk all 0.1-10 [7,538 kB]\n",
            "Get:15 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-hi all 0.1-10 [16.8 kB]\n",
            "Get:16 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festvox-mr-nsk all 0.1-10 [7,543 kB]\n",
            "Get:17 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-mr all 0.1-10 [17.3 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festvox-te-nsk all 0.3.3-5 [7,547 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-te all 0.3.3-5 [13.3 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festlex-cmu all 2.4-2 [895 kB]\n",
            "Get:21 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festlex-poslex all 2.4-1 [186 kB]\n",
            "Get:22 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festvox-ca-ona-hts all 1.3-3 [3,583 kB]\n",
            "Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festvox-czech-ph all 0.1-6 [9,573 kB]\n",
            "Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libestools-dev amd64 1:2.5.0-12 [1,408 kB]\n",
            "Get:25 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libopencore-amrnb0 amd64 0.1.5-1 [94.8 kB]\n",
            "Get:26 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libopencore-amrwb0 amd64 0.1.5-1 [49.1 kB]\n",
            "Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libsox3 amd64 14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1 [240 kB]\n",
            "Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libsox-fmt-alsa amd64 14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1 [11.2 kB]\n",
            "Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 libwavpack1 amd64 5.4.0-1build2 [83.7 kB]\n",
            "Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libsox-fmt-base amd64 14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1 [33.7 kB]\n",
            "Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 sox amd64 14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1 [104 kB]\n",
            "Get:32 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festival-dev amd64 1:2.5.0-8 [606 kB]\n",
            "Get:33 http://archive.ubuntu.com/ubuntu jammy/universe amd64 festvox-kallpc16k all 2.4-1 [3,614 kB]\n",
            "Fetched 50.5 MB in 1s (43.4 MB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 33.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package sgml-base.\n",
            "(Reading database ... 122255 files and directories currently installed.)\n",
            "Preparing to unpack .../00-sgml-base_1.30_all.deb ...\n",
            "Unpacking sgml-base (1.30) ...\n",
            "Selecting previously unselected package libatopology2:amd64.\n",
            "Preparing to unpack .../01-libatopology2_1.2.6.1-1ubuntu1_amd64.deb ...\n",
            "Unpacking libatopology2:amd64 (1.2.6.1-1ubuntu1) ...\n",
            "Selecting previously unselected package libfftw3-single3:amd64.\n",
            "Preparing to unpack .../02-libfftw3-single3_3.3.8-2ubuntu8_amd64.deb ...\n",
            "Unpacking libfftw3-single3:amd64 (3.3.8-2ubuntu8) ...\n",
            "Selecting previously unselected package alsa-utils.\n",
            "Preparing to unpack .../03-alsa-utils_1.2.6-1ubuntu1_amd64.deb ...\n",
            "Unpacking alsa-utils (1.2.6-1ubuntu1) ...\n",
            "Selecting previously unselected package libuuid-perl.\n",
            "Preparing to unpack .../04-libuuid-perl_0.28-1build4_amd64.deb ...\n",
            "Unpacking libuuid-perl (0.28-1build4) ...\n",
            "Selecting previously unselected package libyaml-tiny-perl.\n",
            "Preparing to unpack .../05-libyaml-tiny-perl_1.73-1_all.deb ...\n",
            "Unpacking libyaml-tiny-perl (1.73-1) ...\n",
            "Selecting previously unselected package doc-base.\n",
            "Preparing to unpack .../06-doc-base_0.11.1_all.deb ...\n",
            "Unpacking doc-base (0.11.1) ...\n",
            "Selecting previously unselected package libestools2.5:amd64.\n",
            "Preparing to unpack .../07-libestools2.5_1%3a2.5.0-12_amd64.deb ...\n",
            "Unpacking libestools2.5:amd64 (1:2.5.0-12) ...\n",
            "Selecting previously unselected package festival.\n",
            "Preparing to unpack .../08-festival_1%3a2.5.0-8_amd64.deb ...\n",
            "Unpacking festival (1:2.5.0-8) ...\n",
            "Selecting previously unselected package festival-ca.\n",
            "Preparing to unpack .../09-festival-ca_3.0.6-2_all.deb ...\n",
            "Unpacking festival-ca (3.0.6-2) ...\n",
            "Selecting previously unselected package festival-czech.\n",
            "Preparing to unpack .../10-festival-czech_0.3-5_all.deb ...\n",
            "Unpacking festival-czech (0.3-5) ...\n",
            "Selecting previously unselected package festival-doc.\n",
            "Preparing to unpack .../11-festival-doc_1%3a2.5.0-8_all.deb ...\n",
            "Unpacking festival-doc (1:2.5.0-8) ...\n",
            "Selecting previously unselected package festival-freebsoft-utils.\n",
            "Preparing to unpack .../12-festival-freebsoft-utils_0.10-9_all.deb ...\n",
            "Unpacking festival-freebsoft-utils (0.10-9) ...\n",
            "Selecting previously unselected package festvox-hi-nsk.\n",
            "Preparing to unpack .../13-festvox-hi-nsk_0.1-10_all.deb ...\n",
            "Unpacking festvox-hi-nsk (0.1-10) ...\n",
            "Selecting previously unselected package festival-hi.\n",
            "Preparing to unpack .../14-festival-hi_0.1-10_all.deb ...\n",
            "Unpacking festival-hi (0.1-10) ...\n",
            "Selecting previously unselected package festvox-mr-nsk.\n",
            "Preparing to unpack .../15-festvox-mr-nsk_0.1-10_all.deb ...\n",
            "Unpacking festvox-mr-nsk (0.1-10) ...\n",
            "Selecting previously unselected package festival-mr.\n",
            "Preparing to unpack .../16-festival-mr_0.1-10_all.deb ...\n",
            "Unpacking festival-mr (0.1-10) ...\n",
            "Selecting previously unselected package festvox-te-nsk.\n",
            "Preparing to unpack .../17-festvox-te-nsk_0.3.3-5_all.deb ...\n",
            "Unpacking festvox-te-nsk (0.3.3-5) ...\n",
            "Selecting previously unselected package festival-te.\n",
            "Preparing to unpack .../18-festival-te_0.3.3-5_all.deb ...\n",
            "Unpacking festival-te (0.3.3-5) ...\n",
            "Selecting previously unselected package festlex-cmu.\n",
            "Preparing to unpack .../19-festlex-cmu_2.4-2_all.deb ...\n",
            "Unpacking festlex-cmu (2.4-2) ...\n",
            "Selecting previously unselected package festlex-poslex.\n",
            "Preparing to unpack .../20-festlex-poslex_2.4-1_all.deb ...\n",
            "Unpacking festlex-poslex (2.4-1) ...\n",
            "Selecting previously unselected package festvox-ca-ona-hts.\n",
            "Preparing to unpack .../21-festvox-ca-ona-hts_1.3-3_all.deb ...\n",
            "Unpacking festvox-ca-ona-hts (1.3-3) ...\n",
            "Selecting previously unselected package festvox-czech-ph.\n",
            "Preparing to unpack .../22-festvox-czech-ph_0.1-6_all.deb ...\n",
            "Unpacking festvox-czech-ph (0.1-6) ...\n",
            "Selecting previously unselected package libestools-dev.\n",
            "Preparing to unpack .../23-libestools-dev_1%3a2.5.0-12_amd64.deb ...\n",
            "Unpacking libestools-dev (1:2.5.0-12) ...\n",
            "Selecting previously unselected package libopencore-amrnb0:amd64.\n",
            "Preparing to unpack .../24-libopencore-amrnb0_0.1.5-1_amd64.deb ...\n",
            "Unpacking libopencore-amrnb0:amd64 (0.1.5-1) ...\n",
            "Selecting previously unselected package libopencore-amrwb0:amd64.\n",
            "Preparing to unpack .../25-libopencore-amrwb0_0.1.5-1_amd64.deb ...\n",
            "Unpacking libopencore-amrwb0:amd64 (0.1.5-1) ...\n",
            "Selecting previously unselected package libsox3:amd64.\n",
            "Preparing to unpack .../26-libsox3_14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1_amd64.deb ...\n",
            "Unpacking libsox3:amd64 (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package libsox-fmt-alsa:amd64.\n",
            "Preparing to unpack .../27-libsox-fmt-alsa_14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1_amd64.deb ...\n",
            "Unpacking libsox-fmt-alsa:amd64 (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package libwavpack1:amd64.\n",
            "Preparing to unpack .../28-libwavpack1_5.4.0-1build2_amd64.deb ...\n",
            "Unpacking libwavpack1:amd64 (5.4.0-1build2) ...\n",
            "Selecting previously unselected package libsox-fmt-base:amd64.\n",
            "Preparing to unpack .../29-libsox-fmt-base_14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1_amd64.deb ...\n",
            "Unpacking libsox-fmt-base:amd64 (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package sox.\n",
            "Preparing to unpack .../30-sox_14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1_amd64.deb ...\n",
            "Unpacking sox (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Selecting previously unselected package festival-dev.\n",
            "Preparing to unpack .../31-festival-dev_1%3a2.5.0-8_amd64.deb ...\n",
            "Unpacking festival-dev (1:2.5.0-8) ...\n",
            "Selecting previously unselected package festvox-kallpc16k.\n",
            "Preparing to unpack .../32-festvox-kallpc16k_2.4-1_all.deb ...\n",
            "Unpacking festvox-kallpc16k (2.4-1) ...\n",
            "Setting up libfftw3-single3:amd64 (3.3.8-2ubuntu8) ...\n",
            "Setting up libuuid-perl (0.28-1build4) ...\n",
            "Setting up libsox3:amd64 (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Setting up libestools2.5:amd64 (1:2.5.0-12) ...\n",
            "Setting up libyaml-tiny-perl (1.73-1) ...\n",
            "Setting up festvox-hi-nsk (0.1-10) ...\n",
            "Setting up libopencore-amrwb0:amd64 (0.1.5-1) ...\n",
            "Setting up festvox-mr-nsk (0.1-10) ...\n",
            "Setting up libsox-fmt-alsa:amd64 (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Setting up libwavpack1:amd64 (5.4.0-1build2) ...\n",
            "Setting up festvox-te-nsk (0.3.3-5) ...\n",
            "Setting up libopencore-amrnb0:amd64 (0.1.5-1) ...\n",
            "Setting up libestools-dev (1:2.5.0-12) ...\n",
            "Setting up sgml-base (1.30) ...\n",
            "Setting up libsox-fmt-base:amd64 (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Setting up festival-dev (1:2.5.0-8) ...\n",
            "Setting up libatopology2:amd64 (1.2.6.1-1ubuntu1) ...\n",
            "Setting up doc-base (0.11.1) ...\n",
            "Registering 22 doc-base files...\n",
            "Error in `/usr/share/doc-base/base-passwd.users-and-groups', line 13: all `Format' sections are invalid.\n",
            "Note: `install-docs --verbose --check file_name' may give more details about the above error.\n",
            "Setting up festival-doc (1:2.5.0-8) ...\n",
            "Setting up alsa-utils (1.2.6-1ubuntu1) ...\n",
            "Setting up festival (1:2.5.0-8) ...\n",
            "Setting up sox (14.4.2+git20190427-2+deb11u2ubuntu0.22.04.1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for sgml-base (1.30) ...\n",
            "Setting up festlex-poslex (2.4-1) ...\n",
            "Setting up festival-te (0.3.3-5) ...\n",
            "Setting up festival-hi (0.1-10) ...\n",
            "Setting up festival-mr (0.1-10) ...\n",
            "Setting up festlex-cmu (2.4-2) ...\n",
            "Setting up festival-czech (0.3-5) ...\n",
            "Setting up festvox-czech-ph (0.1-6) ...\n",
            "Setting up festival-ca (3.0.6-2) ...\n",
            "Setting up festival-freebsoft-utils (0.10-9) ...\n",
            "Setting up festvox-kallpc16k (2.4-1) ...\n",
            "Setting up festvox-ca-ona-hts (1.3-3) ...\n",
            "[INFO] B.3 Installing festival... done\n",
            "[INFO] B.4 Installing common libs using apt-get...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "build-essential is already the newest version (12.9ubuntu3).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 51 not upgraded.\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "libasound2-dev is already the newest version (1.2.6.1-1ubuntu1).\n",
            "libsndfile1-dev is already the newest version (1.0.31-2ubuntu0.1).\n",
            "Suggested packages:\n",
            "  libaudio2 libsndio6.1\n",
            "The following NEW packages will be installed:\n",
            "  flac libao-common libao4 vorbis-tools\n",
            "0 upgraded, 4 newly installed, 0 to remove and 51 not upgraded.\n",
            "Need to get 367 kB of archives.\n",
            "After this operation, 1,377 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 flac amd64 1.3.3-2ubuntu0.2 [130 kB]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libao-common all 1.2.2+20180113-1.1ubuntu3 [6,568 B]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 libao4 amd64 1.2.2+20180113-1.1ubuntu3 [35.2 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu jammy/universe amd64 vorbis-tools amd64 1.4.2-1 [195 kB]\n",
            "Fetched 367 kB in 0s (1,239 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 4.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package flac.\n",
            "(Reading database ... 123430 files and directories currently installed.)\n",
            "Preparing to unpack .../flac_1.3.3-2ubuntu0.2_amd64.deb ...\n",
            "Unpacking flac (1.3.3-2ubuntu0.2) ...\n",
            "Selecting previously unselected package libao-common.\n",
            "Preparing to unpack .../libao-common_1.2.2+20180113-1.1ubuntu3_all.deb ...\n",
            "Unpacking libao-common (1.2.2+20180113-1.1ubuntu3) ...\n",
            "Selecting previously unselected package libao4:amd64.\n",
            "Preparing to unpack .../libao4_1.2.2+20180113-1.1ubuntu3_amd64.deb ...\n",
            "Unpacking libao4:amd64 (1.2.2+20180113-1.1ubuntu3) ...\n",
            "Selecting previously unselected package vorbis-tools.\n",
            "Preparing to unpack .../vorbis-tools_1.4.2-1_amd64.deb ...\n",
            "Unpacking vorbis-tools (1.4.2-1) ...\n",
            "Setting up libao-common (1.2.2+20180113-1.1ubuntu3) ...\n",
            "Setting up flac (1.3.3-2ubuntu0.2) ...\n",
            "Setting up libao4:amd64 (1.2.2+20180113-1.1ubuntu3) ...\n",
            "Setting up vorbis-tools (1.4.2-1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n",
            "\n",
            "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n",
            "\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Note, selecting 'libxslt1-dev' instead of 'libxslt-dev'\n",
            "libxml2-dev is already the newest version (2.9.13+dfsg-1ubuntu0.4).\n",
            "zlib1g-dev is already the newest version (1:1.2.11.dfsg-2ubuntu9.2).\n",
            "zlib1g-dev set to manually installed.\n",
            "The following NEW packages will be installed:\n",
            "  libxslt1-dev\n",
            "0 upgraded, 1 newly installed, 0 to remove and 51 not upgraded.\n",
            "Need to get 219 kB of archives.\n",
            "After this operation, 2,058 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libxslt1-dev amd64 1.1.34-4ubuntu0.22.04.1 [219 kB]\n",
            "Fetched 219 kB in 0s (1,358 kB/s)\n",
            "debconf: unable to initialize frontend: Dialog\n",
            "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 1.)\n",
            "debconf: falling back to frontend: Readline\n",
            "debconf: unable to initialize frontend: Readline\n",
            "debconf: (This frontend requires a controlling tty.)\n",
            "debconf: falling back to frontend: Teletype\n",
            "dpkg-preconfigure: unable to re-open stdin: \n",
            "Selecting previously unselected package libxslt1-dev:amd64.\n",
            "(Reading database ... 123496 files and directories currently installed.)\n",
            "Preparing to unpack .../libxslt1-dev_1.1.34-4ubuntu0.22.04.1_amd64.deb ...\n",
            "Unpacking libxslt1-dev:amd64 (1.1.34-4ubuntu0.22.04.1) ...\n",
            "Setting up libxslt1-dev:amd64 (1.1.34-4ubuntu0.22.04.1) ...\n",
            "Processing triggers for man-db (2.10.2-1) ...\n",
            "Processing triggers for doc-base (0.11.1) ...\n",
            "Processing 1 added doc-base file...\n",
            "Reading package lists... Done\n",
            "Building dependency tree... Done\n",
            "Reading state information... Done\n",
            "Package python-dev is not available, but is referred to by another package.\n",
            "This may mean that the package is missing, has been obsoleted, or\n",
            "is only available from another source\n",
            "However the following packages replace it:\n",
            "  python2-dev python2 python-dev-is-python3\n",
            "\n",
            "E: Package 'python-dev' has no installation candidate\n",
            "[INFO] B.4 Installing common libs using apt-get... done\n",
            "[INFO] Congratulations, now you can use aeneas!\n"
          ]
        }
      ],
      "source": [
        "!wget https://raw.githubusercontent.com/readbeyond/aeneas/master/install_dependencies.sh\n",
        "!bash install_dependencies.sh"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SpjTiZH71Pix",
        "outputId": "0c449734-a794-4a80-cca9-9670252a5551"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'aeneas'...\n",
            "remote: Enumerating objects: 5636, done.\u001b[K\n",
            "remote: Counting objects: 100% (19/19), done.\u001b[K\n",
            "remote: Compressing objects: 100% (18/18), done.\u001b[K\n",
            "remote: Total 5636 (delta 0), reused 10 (delta 0), pack-reused 5617\u001b[K\n",
            "Receiving objects: 100% (5636/5636), 29.86 MiB | 22.90 MiB/s, done.\n",
            "Resolving deltas: 100% (4272/4272), done.\n",
            "Requirement already satisfied: BeautifulSoup4>=4.5.1 in /usr/local/lib/python3.10/dist-packages (from -r /content/aeneas/requirements.txt (line 1)) (4.12.3)\n",
            "Requirement already satisfied: lxml>=3.6.0 in /usr/local/lib/python3.10/dist-packages (from -r /content/aeneas/requirements.txt (line 2)) (4.9.4)\n",
            "Requirement already satisfied: numpy>=1.9 in /usr/local/lib/python3.10/dist-packages (from -r /content/aeneas/requirements.txt (line 3)) (1.22.4)\n",
            "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from BeautifulSoup4>=4.5.1->-r /content/aeneas/requirements.txt (line 1)) (2.5)\n",
            "\u001b[39mrunning build_ext\u001b[0m\n",
            "\u001b[39mbuilding 'aeneas.cdtw.cdtw' extension\u001b[0m\n",
            "\u001b[39mWarning: Can't read registry to find the necessary compiler setting\n",
            "Make sure that Python modules winreg, win32api or win32con are installed.\u001b[0m\n",
            "\u001b[39mINFO: C compiler: x86_64-linux-gnu-gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC\n",
            "\u001b[0m\n",
            "\u001b[39mcreating build\u001b[0m\n",
            "\u001b[39mcreating build/temp.linux-x86_64-cpython-310\u001b[0m\n",
            "\u001b[39mcreating build/temp.linux-x86_64-cpython-310/aeneas\u001b[0m\n",
            "\u001b[39mcreating build/temp.linux-x86_64-cpython-310/aeneas/cdtw\u001b[0m\n",
            "\u001b[39mcreating build/temp.linux-x86_64-cpython-310/aeneas/cint\u001b[0m\n",
            "\u001b[39mINFO: compile options: '-I/usr/local/lib/python3.10/dist-packages/numpy/core/include -I['/usr/local/lib/python3.10/dist-packages/numpy/core/include'] -I/usr/include/python3.10 -c'\u001b[0m\n",
            "\u001b[39mINFO: x86_64-linux-gnu-gcc: aeneas/cdtw/cdtw_func.c\u001b[0m\n",
            "\u001b[39mINFO: x86_64-linux-gnu-gcc: aeneas/cdtw/cdtw_py.c\u001b[0m\n",
            "\u001b[01m\u001b[Kcc1:\u001b[m\u001b[K \u001b[01;31m\u001b[Kfatal error: \u001b[m\u001b[Kaeneas/cdtw/cdtw_py.c: No such file or directory\n",
            "compilation terminated.\n",
            "\u001b[01m\u001b[Kcc1:\u001b[m\u001b[K \u001b[01;31m\u001b[Kfatal error: \u001b[m\u001b[Kaeneas/cdtw/cdtw_func.c: No such file or directory\n",
            "compilation terminated.\n",
            "\u001b[39mINFO: x86_64-linux-gnu-gcc: aeneas/cint/cint.c\u001b[0m\n",
            "\u001b[01m\u001b[Kcc1:\u001b[m\u001b[K \u001b[01;31m\u001b[Kfatal error: \u001b[m\u001b[Kaeneas/cint/cint.c: No such file or directory\n",
            "compilation terminated.\n",
            "error: Command \"x86_64-linux-gnu-gcc -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC -I/usr/local/lib/python3.10/dist-packages/numpy/core/include -I['/usr/local/lib/python3.10/dist-packages/numpy/core/include'] -I/usr/include/python3.10 -c aeneas/cdtw/cdtw_func.c -o build/temp.linux-x86_64-cpython-310/aeneas/cdtw/cdtw_func.o\" failed with exit status 1\n",
            "\u001b[92m[INFO] aeneas         OK\u001b[0m\n",
            "\u001b[92m[INFO] ffprobe        OK\u001b[0m\n",
            "\u001b[92m[INFO] ffmpeg         OK\u001b[0m\n",
            "\u001b[92m[INFO] espeak         OK\u001b[0m\n",
            "\u001b[92m[INFO] aeneas.tools   OK\u001b[0m\n",
            "\u001b[92m[INFO] shell encoding OK\u001b[0m\n",
            "\u001b[93m[WARN] aeneas.cdtw    NOT AVAILABLE\u001b[0m\n",
            "[INFO]   You can still run aeneas but it will be significantly slower\n",
            "[INFO]   Please refer to the installation documentation for details\n",
            "\u001b[93m[WARN] aeneas.cmfcc   NOT AVAILABLE\u001b[0m\n",
            "[INFO]   You can still run aeneas but it will be significantly slower\n",
            "[INFO]   Please refer to the installation documentation for details\n",
            "\u001b[93m[WARN] aeneas.cew     NOT AVAILABLE\u001b[0m\n",
            "[INFO]   You can still run aeneas but it will be a bit slower\n",
            "[INFO]   Please refer to the installation documentation for details\n",
            "\u001b[93m[WARN] All required dependencies are met but at least one Python C extension is not available\u001b[0m\n",
            "\u001b[93m[WARN] You can still run aeneas but it will be slower\u001b[0m\n",
            "\u001b[93m[WARN] Enjoy running aeneas!\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/ReadBeyond/aeneas.git\n",
        "!cd /content/aeneas\n",
        "!sudo pip install -r /content/aeneas/requirements.txt\n",
        "!python /content/aeneas/setup.py build_ext --inplace\n",
        "!python /content/aeneas/aeneas_check_setup.py"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "roqTZsHw1Sxf",
        "outputId": "5c24d091-1d60-4d76-9f8e-19ddc839cfd2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.5/5.5 MB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for aeneas (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "!pip install -q aeneas"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9mZwov_K1ULt",
        "outputId": "ba39ff2b-2681-4573-9bce-a9e47c6c7a9a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[92m[INFO] ffprobe        OK\u001b[0m\n",
            "\u001b[92m[INFO] ffmpeg         OK\u001b[0m\n",
            "\u001b[92m[INFO] espeak         OK\u001b[0m\n",
            "\u001b[92m[INFO] aeneas.tools   OK\u001b[0m\n",
            "\u001b[93m[WARN] shell encoding WARNING\u001b[0m\n",
            "\u001b[93m[WARN]   The default input encoding of your shell is not UTF-8\u001b[0m\n",
            "\u001b[93m[WARN]   The default output encoding of your shell is not UTF-8\u001b[0m\n",
            "[INFO]   If you plan to use aeneas on the command line,\n",
            "[INFO]   you might want to 'export PYTHONIOENCODING=UTF-8' in your shell\n",
            "\u001b[92m[INFO] aeneas.cdtw    AVAILABLE\u001b[0m\n",
            "\u001b[92m[INFO] aeneas.cmfcc   AVAILABLE\u001b[0m\n",
            "\u001b[92m[INFO] aeneas.cew     AVAILABLE\u001b[0m\n",
            "\u001b[92m[INFO] All required dependencies are met and all available Python C extensions are working\u001b[0m\n"
          ]
        }
      ],
      "source": [
        "!python -m aeneas.diagnostics"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Import Libraries"
      ],
      "metadata": {
        "id": "h65OI0SnxDxd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import re\n",
        "import subprocess\n",
        "import shutil\n",
        "import json\n",
        "from functools import reduce\n",
        "from pydub import AudioSegment\n",
        "from hazm import Normalizer, sent_tokenize, word_tokenize\n",
        "from parsi_io.parsi_io.modules.number_extractor import NumberExtractor\n",
        "from parsi_io.parsi_io.modules.convert_number_to_text import ConvertNumberToText"
      ],
      "metadata": {
        "id": "oSUSv3-kxGHt"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Convert Audio to Mono Mp3"
      ],
      "metadata": {
        "id": "3LrvXJuQXMz-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "raw_data_dir = \"raw-data/\"\n",
        "\n",
        "processed_data_dir = \"processed-data/\"\n",
        "os.makedirs(processed_data_dir, exist_ok=True)"
      ],
      "metadata": {
        "id": "I0v6BaL8WH2z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def convert_audio_to_mono_mp3(input_file_path, output_file_path):\n",
        "    input_file_name = input_file_path.split('/')[-1].split('.')[0]\n",
        "\n",
        "    # Load the audio file\n",
        "    sound = AudioSegment.from_file(input_file_path)\n",
        "\n",
        "    # Convert stereo to mono\n",
        "    sound = sound.set_channels(1)\n",
        "\n",
        "    # Export the audio in MP3 format\n",
        "    sound.export(output_file_path, format=\"mp3\")\n",
        "\n",
        "    return output_file_path\n"
      ],
      "metadata": {
        "id": "AQ3qtfH7X6w2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Iterate over all.m4a files in the source directory\n",
        "for filename in os.listdir(raw_data_dir):\n",
        "    if filename.endswith('.m4a'):\n",
        "        # Construct full file path\n",
        "        source_file_path = os.path.join(raw_data_dir, filename)\n",
        "        # Construct destination file path\n",
        "        destination_file_path = os.path.join(processed_data_dir, filename.replace('.m4a', '.mp3'))\n",
        "\n",
        "        convert_audio_to_mono_mp3(source_file_path, destination_file_path)\n",
        "        print(f'Converted {filename} to MP3.')\n",
        "\n",
        "print('All.m4a files have been converted to.mp3.')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BxqH91ogWWvx",
        "outputId": "bb5917a5-e1d9-4d26-fe0c-fda66c28286b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Converted 2.m4a to MP3.\n",
            "Converted 1.m4a to MP3.\n",
            "Converted 3.m4a to MP3.\n",
            "Converted 4.m4a to MP3.\n",
            "Converted 9.m4a to MP3.\n",
            "Converted 12.m4a to MP3.\n",
            "Converted 14.m4a to MP3.\n",
            "Converted 16.m4a to MP3.\n",
            "Converted 20.m4a to MP3.\n",
            "Converted 22.m4a to MP3.\n",
            "Converted 24.m4a to MP3.\n",
            "Converted 26.m4a to MP3.\n",
            "Converted 30.m4a to MP3.\n",
            "Converted 40.m4a to MP3.\n",
            "Converted 50.m4a to MP3.\n",
            "Converted 52.m4a to MP3.\n",
            "Converted 54.m4a to MP3.\n",
            "Converted 56.m4a to MP3.\n",
            "Converted 64.m4a to MP3.\n",
            "Converted 72.m4a to MP3.\n",
            "Converted 76.m4a to MP3.\n",
            "Converted 101.m4a to MP3.\n",
            "Converted 103.m4a to MP3.\n",
            "Converted 107.m4a to MP3.\n",
            "Converted 111.m4a to MP3.\n",
            "All.m4a files have been converted to.mp3.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Process Text"
      ],
      "metadata": {
        "id": "_eFddpRKYetZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Normalization"
      ],
      "metadata": {
        "id": "EjNPpBz1yhoi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "normalizer = Normalizer()\n",
        "\n",
        "def normalize_text(text):\n",
        "  return normalizer.normalize(text)"
      ],
      "metadata": {
        "id": "MupcUcpbytQq"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Symbol Substitution\n",
        "This step is designed to unify various forms of symbols into their more commonly used counterparts."
      ],
      "metadata": {
        "id": "ard6ph0Uy0dP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "substitution_dict = {'ﯽ': 'ی', '—': '–', '\\u200f': '\\u200c', '\\xad': '\\u200c', '\\u200e': '\\u200c', '\\u200d': '\\u200c'}\n",
        "\n",
        "def substitute_symbols(text):\n",
        "    translation_table = str.maketrans(substitution_dict)\n",
        "    substituted_text = text.translate(translation_table)\n",
        "    return substituted_text"
      ],
      "metadata": {
        "id": "fNfrmnVgzQV3"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove In-text References\n",
        "This step is designed to remove the references that come inside the text but are not read aloud. For example:\n",
        "> They have introduced a new tool [1] which ..."
      ],
      "metadata": {
        "id": "rfTD3b0nz4Lr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_inline_references(text):\n",
        "    # Define pattern to match references like \"[NUM]\"\n",
        "    pattern_fa = r\"\\[\\d+\\]|\\[۰-۹]+\\]\"\n",
        "    pattern_en = r\"\\[\\d+\\]|\\[0-9]+\\]\"\n",
        "\n",
        "    # Use regular expression to remove references\n",
        "    text_without_refs_fa = re.sub(pattern_fa, \" \", text)\n",
        "    text_without_refs_en = re.sub(pattern_en, \" \", text_without_refs_fa)\n",
        "\n",
        "    return text_without_refs_en"
      ],
      "metadata": {
        "id": "Ftu0Vajm0NQZ"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Reference Lines\n",
        "This step is designed to remove the references that come at the end of the text but are not read aloud. For example:\n",
        "> [1] Roshan-AI. Hazm. https://www.roshan-ai.ir/hazm/docs/index.html. Accessed:\n",
        "May 3, 2024.\n",
        ">\n",
        "> [2] ...\n",
        "\n"
      ],
      "metadata": {
        "id": "01MnXk8g0Qbc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_references_lines(text):\n",
        "    # Define pattern to match references like \"[NUM] \"\n",
        "    pattern_fa = r\"^\\s*\\[\\d+\\]|\\[۰-۹]+\\]\"\n",
        "    pattern_en = r\"^\\s*\\[\\d+\\]|\\[0-9]+\\]\"\n",
        "\n",
        "    # Split text into lines\n",
        "    lines = text.split('\\n')\n",
        "\n",
        "    # Remove lines starting with references\n",
        "    cleaned_lines = [line for line in lines if not re.match(pattern_fa, line.strip()) and not re.match(pattern_en, line.strip())]\n",
        "\n",
        "    # Join cleaned lines back into text\n",
        "    cleaned_text = '\\n'.join(cleaned_lines)\n",
        "\n",
        "    return cleaned_text"
      ],
      "metadata": {
        "id": "L7yk3Ube0nU_"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Link Lines\n",
        "This step is designed to remove the links and urls that come at the end of the text but are not read aloud. For example:\n",
        "> Resources:\n",
        ">\n",
        "> https://www.roshan-ai.ir/hazm/docs/index.html\n",
        ">\n",
        "> https://virgool.io/\n",
        ">\n",
        "> ..."
      ],
      "metadata": {
        "id": "9-jcYsWn0t9n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_link_lines(text):\n",
        "    # Define the pattern to match lines starting with http or www\n",
        "    pattern = r\"^\\s*(?:http|www)\"\n",
        "\n",
        "    # Split text into lines\n",
        "    lines = text.split('\\n')\n",
        "\n",
        "    # Remove lines starting with link\n",
        "    cleaned_lines = [line for line in lines if not re.match(pattern, line.strip())]\n",
        "\n",
        "    # Join cleaned lines back into text\n",
        "    cleaned_text = '\\n'.join(cleaned_lines)\n",
        "\n",
        "    return cleaned_text"
      ],
      "metadata": {
        "id": "U41eap2C0oWn"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Convert Numbers to Text\n",
        "This step is designed to convert the numbers in digit format into their spoken version. For example:\n",
        "\n",
        "> 22 → twenty two"
      ],
      "metadata": {
        "id": "08-yP9Hb1OV9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "persian_digits_pattern = re.compile(r'[۰۱۲۳۴۵۶۷۸۹0123456789]')\n",
        "num2text = ConvertNumberToText()\n",
        "extractor = NumberExtractor()"
      ],
      "metadata": {
        "id": "ZcVnmGRx1oBS"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def replace_numbers_with_text(text):\n",
        "  # Find all number spans in the text\n",
        "  number_spans = extractor.run(text)\n",
        "\n",
        "  # Filter out spans that contain digits\n",
        "  filtered_spans = [span for span in number_spans if persian_digits_pattern.search(span['phrase'])]\n",
        "\n",
        "  # Convert the filtered numbers to text and replace them in the text\n",
        "  offset = 0  # Track the offset due to previous replacements\n",
        "  for span in filtered_spans:\n",
        "      start, end = span['span']\n",
        "      start -= offset  # Adjust start position based on previous replacements\n",
        "      end -= offset  # Adjust end position based on previous replacements\n",
        "      number_text = span['phrase']\n",
        "      number_value = span['value']\n",
        "\n",
        "      # Convert the number to text\n",
        "      text_value = num2text.run(str(number_value))\n",
        "\n",
        "      # Replace the number in the text with its textual equivalent\n",
        "      text = text[:start] + text_value + text[end:]\n",
        "\n",
        "      # Update the offset\n",
        "      offset += len(number_text) - len(text_value)\n",
        "\n",
        "  return text"
      ],
      "metadata": {
        "id": "HylkKIWp1saj"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Symbols\n",
        "This step is designed to remove some of the symbols that are not very common or do not affect the TTS-ASR models' outputs. This helps simplify the input to the models."
      ],
      "metadata": {
        "id": "Zyghd-a62tid"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "symbols_to_remove = \"«»*[]\\\"'^&<>{}|٫《》•\\x9d\\u200b\\x7f\"\n",
        "\n",
        "def remove_symbols(text):\n",
        "    pattern = \"[\" + re.escape(symbols_to_remove) + \"]\"\n",
        "    return re.sub(pattern, ' ', text)"
      ],
      "metadata": {
        "id": "VlA-vsW-29Vq"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Remove Extra White Spaces\n",
        "This step is designed to remove extra white spaces inluding multiple consequent white spaces and new lines."
      ],
      "metadata": {
        "id": "AGc1r6vx3EdV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def remove_extra_white_spaces(text):\n",
        "    cleaned_text = re.sub(r'\\s+', ' ', text)\n",
        "    return cleaned_text.strip()"
      ],
      "metadata": {
        "id": "DnJGXdap3Zc0"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Full Pipeline\n",
        "Here we define the complete text processing pipelin and the processing code."
      ],
      "metadata": {
        "id": "XwjdlP2o3Zya"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline = [\n",
        "    normalize_text,\n",
        "    substitute_symbols,\n",
        "    remove_inline_references,\n",
        "    remove_references_lines,\n",
        "    remove_link_lines,\n",
        "    replace_numbers_with_text,\n",
        "    remove_symbols,\n",
        "    remove_extra_white_spaces\n",
        "  ]"
      ],
      "metadata": {
        "id": "t9JzXgxM3hQO"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def process_text(input_file_path, output_dir_path):\n",
        "  input_file_name = input_file_path.split('/')[-1].split('.')[0]\n",
        "  output_file_path = os.path.join(output_dir_path, input_file_name + '.txt')\n",
        "\n",
        "  # Check if the output file already exists\n",
        "  if os.path.exists(output_file_path):\n",
        "    print(f\"Skipping file {input_file_name}. Processed text file already exists.\")\n",
        "    return output_file_path\n",
        "\n",
        "  # Apply the text processing pipeline\n",
        "  with open(input_file_path, 'r') as f:\n",
        "    text = reduce(lambda txt, func: func(txt), pipeline, f.read())\n",
        "\n",
        "  # Export the processed text\n",
        "  with open(output_file_path, 'w') as f:\n",
        "     f.write(text)\n",
        "\n",
        "  return output_file_path"
      ],
      "metadata": {
        "id": "2HM4ETq_3kom"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Run the text processing pipeline"
      ],
      "metadata": {
        "id": "QswV18q63pZP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def process_all_text_files(directory_path, output_dir_path):\n",
        "    # Ensure the output directory exists\n",
        "    if not os.path.exists(output_dir_path):\n",
        "        os.makedirs(output_dir_path)\n",
        "\n",
        "    # Iterate over all files in the directory\n",
        "    for filename in os.listdir(directory_path):\n",
        "        if filename.endswith('.txt'):  # Check if the file is a text file\n",
        "            input_file_path = os.path.join(directory_path, filename)\n",
        "            output_file_path = os.path.join(output_dir_path, filename)\n",
        "\n",
        "            # Call the process_text function for each text file\n",
        "            process_text(input_file_path, output_dir_path)"
      ],
      "metadata": {
        "id": "sB-usl1P3vY6"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "process_all_text_files(raw_data_dir, processed_data_dir)"
      ],
      "metadata": {
        "id": "goiie6IqaY4t",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58660ebf-8206-4504-c456-edeb466e00a3"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Skipping file 1. Processed text file already exists.\n",
            "Skipping file 2. Processed text file already exists.\n",
            "Skipping file 3. Processed text file already exists.\n",
            "Skipping file 4. Processed text file already exists.\n",
            "Skipping file 9. Processed text file already exists.\n",
            "Skipping file 12. Processed text file already exists.\n",
            "Skipping file 14. Processed text file already exists.\n",
            "Skipping file 16. Processed text file already exists.\n",
            "Skipping file 20. Processed text file already exists.\n",
            "Skipping file 22. Processed text file already exists.\n",
            "Skipping file 24. Processed text file already exists.\n",
            "Skipping file 26. Processed text file already exists.\n",
            "Skipping file 30. Processed text file already exists.\n",
            "Skipping file 40. Processed text file already exists.\n",
            "Skipping file 50. Processed text file already exists.\n",
            "Skipping file 52. Processed text file already exists.\n",
            "Skipping file 54. Processed text file already exists.\n",
            "Skipping file 56. Processed text file already exists.\n",
            "Skipping file 64. Processed text file already exists.\n",
            "Skipping file 72. Processed text file already exists.\n",
            "Skipping file 76. Processed text file already exists.\n",
            "Skipping file 101. Processed text file already exists.\n",
            "Skipping file 103. Processed text file already exists.\n",
            "Skipping file 107. Processed text file already exists.\n",
            "Skipping file 111. Processed text file already exists.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Forced Alignment\n",
        "Forced alignment is the task of chunking the audio and text files into smaller parts of a few seconds and a few words. The resulting audio-text chunks contain the same content. I.e. the text files are the transcript of the audio files.\n",
        "\n",
        "Here we use the [Aeneas](https://github.com/readbeyond/aeneas) forced alignment tool which is a good choice for audio and text files that are an exact match.\n",
        "\n",
        "This tool requires the text be tokenized to sentences. We use the [Hazm](https://www.roshan-ai.ir/hazm/docs/index.html) sentence tokenizer and [Perpos](https://github.com/mhbashari/perpos) POS Tagger and develop a new sentence tokenization tool that keeps the sentences in a predefined length range."
      ],
      "metadata": {
        "id": "xLwXCW7hYbDE"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zNBFLb7j8F_P"
      },
      "source": [
        "## Sentence Tokenize"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cMT1eKD5v-9t"
      },
      "outputs": [],
      "source": [
        "def contains_word_letter(input_string):\n",
        "    regex_pattern = re.compile(r'\\w+')\n",
        "    return bool(regex_pattern.search(input_string))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The implemented sentence tokenizer first uses the Hazm sentnece tokenizer that mostly tokenizes based on punctuation. It then uses the Perpos POS Tagger to identify VERBS and EZAFE tags. It uses the position of VERB tags to further split the sentences into smaller chunks. This tokenizer considers several criteria during tokenization to output meaningful and well-formed splits:\n",
        "\n",
        "*   It appends all the symbols to the verb of the last sentence. This is because the symbols can affect the pronunciation of the verb (consider how a `?` mark changes intonation).\n",
        "*   It appends the conjunction `و` to the verb of the last sentence. Because the pronunciation of this word is usually integrated with the previous word by a vowel sound `\\o\\` and should not be interrupted.\n",
        "*   It keeps sentences under a maximum length\n",
        "*   It merges small chunks together to keep sentences longer than a minimum length\n",
        "*   It avoids splitting the sentences at the EZAFE tags. This is because these wrods are pronounced connected to the previous words by a vowel `\\e\\` and should not be interrupted."
      ],
      "metadata": {
        "id": "GByUfNUv-gvd"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Y6LB2k28ykrL"
      },
      "outputs": [],
      "source": [
        "def get_sub_sentences(tagged_words, min_split_len=5, max_split_len=12):\n",
        "    i = 0                                                                             # Index of next word in the original sentence\n",
        "    current_words = []\n",
        "    sub_sentences = []\n",
        "\n",
        "    while i < len(tagged_words):\n",
        "      word, pos = tagged_words[i]\n",
        "      current_words.append(word)\n",
        "      i += 1\n",
        "\n",
        "      while i < len(tagged_words) and not contains_word_letter(tagged_words[i][0]):   # Append symbols to previous sentence\n",
        "        current_words.append(tagged_words[i][0])\n",
        "        i += 1\n",
        "\n",
        "      if pos == \"V\" and i < len(tagged_words) and tagged_words[i][0] == \"و\":          # Append \"و\" to previous verb\n",
        "        current_words.append(\"و\")\n",
        "        i += 1\n",
        "\n",
        "      while i < len(tagged_words) and pos.endswith(\"e\"):                              # Append all EZAFE POSes to previous sentence\n",
        "        word, pos = tagged_words[i]\n",
        "        current_words.append(word)\n",
        "        i += 1\n",
        "\n",
        "      if i >= len(tagged_words) or len(current_words) >= max_split_len or (len(current_words) > min_split_len and pos == \"V\"):\n",
        "        sub_sentences.append(' '.join(current_words))\n",
        "        current_words = []\n",
        "\n",
        "    return sub_sentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IrYxESJYynSs"
      },
      "outputs": [],
      "source": [
        "def split_sentences(text, min_split_len=5, max_split_len=12):\n",
        "    print(\"Tokenizing to sentences...\")\n",
        "\n",
        "    hazm_sentences = sent_tokenize(text)\n",
        "    splitted_sentences = []\n",
        "\n",
        "    for sent in hazm_sentences:\n",
        "      words = word_tokenize(sent)\n",
        "      tagged_words = pos_tagger.parse(words)\n",
        "      sub_sentences = get_sub_sentences(tagged_words, min_split_len, max_split_len)\n",
        "      splitted_sentences.extend(sub_sentences)\n",
        "\n",
        "    return splitted_sentences"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4MDSzSoG540k"
      },
      "outputs": [],
      "source": [
        "def write_splitted_text(input_file_path, output_file_path='temp_splitted_text.txt'):\n",
        "  text = \"\"\n",
        "  with open(input_file_path, 'r') as f: text = f.read()\n",
        "  splitted_sentences = split_sentences(text)\n",
        "  splitted_text = '\\n'.join(splitted_sentences)\n",
        "\n",
        "  # Remove _ artifact from hazm word tokenizer\n",
        "  splitted_text = splitted_text.replace('_', '\\u200c')\n",
        "\n",
        "  with open(output_file_path, 'w') as f: f.write(splitted_text)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j1EM85Dv87io"
      },
      "source": [
        "## Forced Alignment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GfIvyQ0V42Tg"
      },
      "outputs": [],
      "source": [
        "def write_forced_alignment_map(audio_file, text_file, output_json='temp_alignment_map.json'):\n",
        "    print(\"Executing forced alignment...\")\n",
        "    command = [\n",
        "        \"python\", \"-m\", \"aeneas.tools.execute_task\",\n",
        "        audio_file,\n",
        "        text_file,\n",
        "        \"task_language=fas|os_task_file_format=json|is_text_type=plain\",\n",
        "        output_json\n",
        "    ]\n",
        "    try:\n",
        "        subprocess.run(command, check=True)\n",
        "        print(\"Forced alignment completed successfully.\")\n",
        "    except subprocess.CalledProcessError as e:\n",
        "        print(\"Error:\", e)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bVuEetVr5WOa"
      },
      "outputs": [],
      "source": [
        "def read_json(json_file):\n",
        "    with open(json_file, 'r') as file:\n",
        "        return json.load(file)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EmAqthxF9n1o"
      },
      "outputs": [],
      "source": [
        "def get_aligned_audio_text(audio_file_path, text_file_path, temp_text_file='temp_splitted_text.txt'):\n",
        "    # Step 1: Split text into sentences and write to temp text file\n",
        "    write_splitted_text(text_file_path, temp_text_file)\n",
        "\n",
        "    # Step 2: Perform forced alignment and write result to temp JSON file\n",
        "    temp_json_file = 'temp_alignment_map.json'\n",
        "    write_forced_alignment_map(audio_file_path, temp_text_file, temp_json_file)\n",
        "\n",
        "    # Step 3: Read JSON file and split audio based on start and end spans\n",
        "    alignment_dict = read_json(temp_json_file)\n",
        "    audio_segments = []\n",
        "    for fragment in alignment_dict['fragments']:\n",
        "        start = float(fragment['begin'])\n",
        "        end = float(fragment['end'])\n",
        "        audio_segments.append((start, end))\n",
        "\n",
        "    # Step 4: Clean up temporary files\n",
        "    os.remove(temp_json_file)\n",
        "\n",
        "    return audio_segments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "imjNpL3_AqnR"
      },
      "source": [
        "## Run Forced Alignment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "C6EvypcPAxgJ"
      },
      "outputs": [],
      "source": [
        "def truncate_audio(audio_file, start, end):\n",
        "    audio = AudioSegment.from_mp3(audio_file)\n",
        "    truncated_audio = audio[float(start) * 1000:float(end) * 1000]\n",
        "    truncated_audio.export(audio_file, format=\"mp3\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hl_-lbMMAtYU"
      },
      "outputs": [],
      "source": [
        "def process_audio_files(source_dir, destination_dir, processed_files_log):\n",
        "    processed_filenames = set()\n",
        "    with open(processed_files_log, 'r') as f:\n",
        "      processed_filenames = set([fname.strip() for fname in f.readlines()])\n",
        "\n",
        "    # Get total number of audio files in the source directory\n",
        "    audio_files = [audio_file for audio_file in os.listdir(source_dir) if audio_file.endswith('.mp3')]\n",
        "    total_files = len(audio_files)\n",
        "\n",
        "    # Iterate through all audio files in the source directory\n",
        "    for idx, audio_file in enumerate(audio_files, start=1):\n",
        "        print(f\"({idx}/{total_files}): Processing file {audio_file}\")\n",
        "        audio_file_path = os.path.join(source_dir, audio_file)\n",
        "        text_file_path = os.path.join(source_dir, audio_file.replace('.mp3', '.txt'))\n",
        "\n",
        "        if not os.path.exists(text_file_path):\n",
        "            print(\"\\x1b[31m\\\"Processed text file does not exist!\\\"\\x1b[0m\")\n",
        "            continue\n",
        "\n",
        "        # Create directory with audio file name in destination directory\n",
        "        audio_name = os.path.splitext(audio_file)[0]\n",
        "        if audio_name in processed_filenames:\n",
        "          print(f\"Skippin audio file {audio_name}... already forced aligned!\")\n",
        "          continue\n",
        "\n",
        "        alignment_dir = os.path.join(destination_dir, audio_name)\n",
        "\n",
        "        # Process audio and text file\n",
        "        temp_text_file = 'temp_splitted_text.txt'\n",
        "        audio_segments = get_aligned_audio_text(audio_file_path, text_file_path, temp_text_file)\n",
        "\n",
        "        print(\"Writing output pairs...\")\n",
        "        os.makedirs(alignment_dir, exist_ok=True)\n",
        "        # Write each line in temp text output file to corresponding txt file\n",
        "        with open(temp_text_file, 'r', encoding='utf-8') as f:\n",
        "            lines = f.readlines()\n",
        "            for idx, line in enumerate(lines):\n",
        "                segment_text_name = f\"{audio_name}-{idx}.txt\"\n",
        "                segment_text_path = os.path.join(alignment_dir, segment_text_name)\n",
        "                with open(segment_text_path, 'w', encoding='utf-8') as segment_file:\n",
        "                    segment_file.write(line.strip())\n",
        "\n",
        "        # Write audio segments to individual files\n",
        "        for idx, segment in enumerate(audio_segments):\n",
        "            start, end = segment\n",
        "            segment_audio_name = f\"{audio_name}-{idx}.mp3\"\n",
        "            segment_audio_path = os.path.join(alignment_dir, segment_audio_name)\n",
        "            # Copy segment from original audio to segment file\n",
        "            shutil.copy(audio_file_path, segment_audio_path)\n",
        "            # Truncate audio segment\n",
        "            truncate_audio(segment_audio_path, start, end)\n",
        "\n",
        "        with open(processed_files_log, 'a') as f: f.write(f'{audio_name}\\n')\n",
        "\n",
        "        os.remove(temp_text_file)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "force_aligned_audio_text_files_dir = \"forced-aligned-data/\"\n",
        "os.makedirs(force_aligned_audio_text_files_dir, exist_ok=True)"
      ],
      "metadata": {
        "id": "PL8SlKlr--AO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# This file keeps track of the files already forced aligned not to repeat the processing\n",
        "forced_aligned_files_log = force_aligned_audio_text_files_dir + 'forced-aligned-files-log.txt'\n",
        "with open(forced_aligned_files_log, 'a') as file: pass"
      ],
      "metadata": {
        "id": "82FWRJVn_RbE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HGrFFHQfEp4m",
        "outputId": "64e84707-5139-477f-e4fc-572f76b44073"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1/25): Processing file 2.mp3\n",
            "Skippin audio file 2... already forced aligned!\n",
            "(2/25): Processing file 1.mp3\n",
            "Skippin audio file 1... already forced aligned!\n",
            "(3/25): Processing file 3.mp3\n",
            "Skippin audio file 3... already forced aligned!\n",
            "(4/25): Processing file 4.mp3\n",
            "Skippin audio file 4... already forced aligned!\n",
            "(5/25): Processing file 9.mp3\n",
            "Skippin audio file 9... already forced aligned!\n",
            "(6/25): Processing file 12.mp3\n",
            "Skippin audio file 12... already forced aligned!\n",
            "(7/25): Processing file 14.mp3\n",
            "Skippin audio file 14... already forced aligned!\n",
            "(8/25): Processing file 16.mp3\n",
            "Skippin audio file 16... already forced aligned!\n",
            "(9/25): Processing file 20.mp3\n",
            "Skippin audio file 20... already forced aligned!\n",
            "(10/25): Processing file 22.mp3\n",
            "Skippin audio file 22... already forced aligned!\n",
            "(11/25): Processing file 24.mp3\n",
            "Skippin audio file 24... already forced aligned!\n",
            "(12/25): Processing file 26.mp3\n",
            "Skippin audio file 26... already forced aligned!\n",
            "(13/25): Processing file 30.mp3\n",
            "Skippin audio file 30... already forced aligned!\n",
            "(14/25): Processing file 40.mp3\n",
            "Skippin audio file 40... already forced aligned!\n",
            "(15/25): Processing file 50.mp3\n",
            "Skippin audio file 50... already forced aligned!\n",
            "(16/25): Processing file 52.mp3\n",
            "Skippin audio file 52... already forced aligned!\n",
            "(17/25): Processing file 54.mp3\n",
            "Skippin audio file 54... already forced aligned!\n",
            "(18/25): Processing file 56.mp3\n",
            "Skippin audio file 56... already forced aligned!\n",
            "(19/25): Processing file 64.mp3\n",
            "Skippin audio file 64... already forced aligned!\n",
            "(20/25): Processing file 72.mp3\n",
            "Tokenizing to sentences...\n",
            "Executing forced alignment...\n",
            "Forced alignment completed successfully.\n",
            "Writing output pairs...\n",
            "(21/25): Processing file 76.mp3\n",
            "Tokenizing to sentences...\n",
            "Executing forced alignment...\n",
            "Forced alignment completed successfully.\n",
            "Writing output pairs...\n",
            "(22/25): Processing file 101.mp3\n",
            "Tokenizing to sentences...\n",
            "Executing forced alignment...\n",
            "Forced alignment completed successfully.\n",
            "Writing output pairs...\n",
            "(23/25): Processing file 103.mp3\n",
            "Tokenizing to sentences...\n",
            "Executing forced alignment...\n",
            "Forced alignment completed successfully.\n",
            "Writing output pairs...\n",
            "(24/25): Processing file 107.mp3\n",
            "Tokenizing to sentences...\n",
            "Executing forced alignment...\n",
            "Forced alignment completed successfully.\n",
            "Writing output pairs...\n",
            "(25/25): Processing file 111.mp3\n",
            "Tokenizing to sentences...\n",
            "Executing forced alignment...\n",
            "Forced alignment completed successfully.\n",
            "Writing output pairs...\n"
          ]
        }
      ],
      "source": [
        "process_audio_files(processed_data_dir, force_aligned_audio_text_files_dir, forced_aligned_files_log)"
      ]
    }
  ]
}