NLGenerationMethods.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "WordGeneration.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMI2LYO0bsZWvD+73KhJOqz"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "961ba477afec4747869f614e9b0aa6e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c62705cc34cc4ff8983e0cfd5f121d3d",
              "IPY_MODEL_9f572612e03f45de83994b3846dff1e5",
              "IPY_MODEL_72807a4c13cc4efbbdba657cee33928a"
            ],
            "layout": "IPY_MODEL_896132a79a0143b1a0b6709084778a73"
          }
        },
        "c62705cc34cc4ff8983e0cfd5f121d3d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_085eccc013dd4e2e83f297efa00ecfb3",
            "placeholder": "​",
            "style": "IPY_MODEL_1114a69a959f4f448eea8eabf8b462ca",
            "value": "Downloading: 100%"
          }
        },
        "9f572612e03f45de83994b3846dff1e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28ea37aea0b84f348e91932582755169",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cf946258f72c40b8a41a1e5abf7f6663",
            "value": 1042301
          }
        },
        "72807a4c13cc4efbbdba657cee33928a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd5b5bbe85b54f58821c41565041b9db",
            "placeholder": "​",
            "style": "IPY_MODEL_f602ec62dbd944c2b152b36f22e66714",
            "value": " 0.99M/0.99M [00:00&lt;00:00, 2.13MB/s]"
          }
        },
        "896132a79a0143b1a0b6709084778a73": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "085eccc013dd4e2e83f297efa00ecfb3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1114a69a959f4f448eea8eabf8b462ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "28ea37aea0b84f348e91932582755169": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf946258f72c40b8a41a1e5abf7f6663": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fd5b5bbe85b54f58821c41565041b9db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f602ec62dbd944c2b152b36f22e66714": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "284e685c81e04a739ae0e527c1c76da5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8aafed3360fb466aafe569881372d49d",
              "IPY_MODEL_0421367c7dd342edb29a54e162613667",
              "IPY_MODEL_9ca0d42e5c364b2fb1c11262f9d1df64"
            ],
            "layout": "IPY_MODEL_d9ef526f23c2472a97eddd20f327365e"
          }
        },
        "8aafed3360fb466aafe569881372d49d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_17bbeb5256ff4a789c733aac347208d3",
            "placeholder": "​",
            "style": "IPY_MODEL_bf1461fdcd354a9ba37f01484809bc2e",
            "value": "Downloading: 100%"
          }
        },
        "0421367c7dd342edb29a54e162613667": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_18cba9a4ac584f66bb8cadd4d09c5564",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8d56757253594f0fbd598993b63bbdfe",
            "value": 456318
          }
        },
        "9ca0d42e5c364b2fb1c11262f9d1df64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_622b365973544aaaae4a04ee954c70fd",
            "placeholder": "​",
            "style": "IPY_MODEL_befe37a2a0044c07be2856382b2894a5",
            "value": " 446k/446k [00:00&lt;00:00, 669kB/s]"
          }
        },
        "d9ef526f23c2472a97eddd20f327365e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "17bbeb5256ff4a789c733aac347208d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf1461fdcd354a9ba37f01484809bc2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "18cba9a4ac584f66bb8cadd4d09c5564": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d56757253594f0fbd598993b63bbdfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "622b365973544aaaae4a04ee954c70fd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "befe37a2a0044c07be2856382b2894a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "64fbb6f473ac4df4bc3440dc049f927d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4f142be048bd4c108b9db563d22393ea",
              "IPY_MODEL_021d05594b1f4e1092a72fc820f70a56",
              "IPY_MODEL_5fe15b22e2d24043adec35f81cfdbb32"
            ],
            "layout": "IPY_MODEL_c652cd148ed5407590914d1a5659b5b7"
          }
        },
        "4f142be048bd4c108b9db563d22393ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d0cd40b9c9c4ec7b9b783205d926659",
            "placeholder": "​",
            "style": "IPY_MODEL_457419cc677a4f13885512a4d797e37c",
            "value": "Downloading: 100%"
          }
        },
        "021d05594b1f4e1092a72fc820f70a56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a5c0dee7b6e49b6a9c1f7827275c7a2",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_92d91fad33a14f5084b4f921f0b1234a",
            "value": 665
          }
        },
        "5fe15b22e2d24043adec35f81cfdbb32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a967532a9eb94f7399dce6b8459c7941",
            "placeholder": "​",
            "style": "IPY_MODEL_a8f72b34266b469191a4ae1eee8aab52",
            "value": " 665/665 [00:00&lt;00:00, 14.9kB/s]"
          }
        },
        "c652cd148ed5407590914d1a5659b5b7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d0cd40b9c9c4ec7b9b783205d926659": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "457419cc677a4f13885512a4d797e37c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2a5c0dee7b6e49b6a9c1f7827275c7a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92d91fad33a14f5084b4f921f0b1234a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a967532a9eb94f7399dce6b8459c7941": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8f72b34266b469191a4ae1eee8aab52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c625f4c86254425fb8fc33dc609df0a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9c5eb85ac1754542b9c929e1230ffb69",
              "IPY_MODEL_112975b9e38249748f893dbabd26f4a3",
              "IPY_MODEL_c0f3fa0739bb421d979f39999a821cf5"
            ],
            "layout": "IPY_MODEL_9b092d0e022e460bac61849735628638"
          }
        },
        "9c5eb85ac1754542b9c929e1230ffb69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fc32a7aacc4a4c769ceb4c1ac72b6f8a",
            "placeholder": "​",
            "style": "IPY_MODEL_fcdb0ce5d197481799a72ae9f353c9f5",
            "value": "Downloading: 100%"
          }
        },
        "112975b9e38249748f893dbabd26f4a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c76a8957ff2144ff8c713fa7df687fb5",
            "max": 548118077,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1047399ff9d946d0b54b7bf7d8a5f6a0",
            "value": 548118077
          }
        },
        "c0f3fa0739bb421d979f39999a821cf5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_38d46fc1c2fe4e5db30eaf77ced45f30",
            "placeholder": "​",
            "style": "IPY_MODEL_1cd5abc30a7e452fbf1eb6de159b5d09",
            "value": " 523M/523M [00:15&lt;00:00, 33.2MB/s]"
          }
        },
        "9b092d0e022e460bac61849735628638": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc32a7aacc4a4c769ceb4c1ac72b6f8a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fcdb0ce5d197481799a72ae9f353c9f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c76a8957ff2144ff8c713fa7df687fb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1047399ff9d946d0b54b7bf7d8a5f6a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "38d46fc1c2fe4e5db30eaf77ced45f30": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cd5abc30a7e452fbf1eb6de159b5d09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Markov Word Generator from Scratch\n",
        "\n",
        "As a starting point to understanding natural language processing and generation, I thought about text generation in a determinstic way, without machine learning, based on the exact words that follow from combination of words before in any given text. And the next words forms part of the combination that then generates the next word, and so on...\n",
        "\n",
        "From my mathematical background, this showed Markov properties which led me to consider how to build a transition matrix from all of this. My exposure to Python and data science at the time was solely with numerical data in Numpy or Pandas and so I approached this problem from that perspective.\n",
        "\n",
        "The first step is to load a corpus of text and clean the data, but preserving grammatical structure such that the Markov process will generate grammatically correct words at each step.\n",
        "\n",
        "I have then created a function to break the text into phrases of any length and the following word/phrase. This essentially creates a dataframe of a feature and a label of 10s of thousands of rows depending on the initial length of the text.\n",
        "\n"
      ],
      "metadata": {
        "id": "XiW0_7NNidVK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ICAvRhkgFY6m",
        "outputId": "5167d521-8de1-4054-8860-36e07f9d1c50"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "9UVuG-cPGEoh"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences"
      ],
      "metadata": {
        "id": "IsZGqITEHOMD"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "root_folder = '/content/drive/My Drive/WordGeneration'"
      ],
      "metadata": {
        "id": "bxQeSI2-GIgF"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data = pd.read_csv(root_folder+'/FOMC2021.txt', sep=\"\\n\")"
      ],
      "metadata": {
        "id": "QDyMJUD8GOCJ"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data.rename(columns={\"Action to Adopt Changes to the Committee's Rules Regarding Availability of Information\": \"text\"},\n",
        "          inplace=True)\n",
        "data[\"text\"] = data[\"text\"].str.replace(\"United States\", \"US\")\n",
        "data[\"text\"] = data[\"text\"].str.replace(\"U.S.\", \"US\")\n",
        "data.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_2Cwg3iuGdMC",
        "outputId": "ddbba66e-8d7a-4038-9a09-ce524e1b3bb1"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 486 entries, 0 to 485\n",
            "Data columns (total 1 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   text    486 non-null    object\n",
            "dtypes: object(1)\n",
            "memory usage: 3.9+ KB\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  after removing the cwd from sys.path.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len_text = 3\n",
        "len_result = 1"
      ],
      "metadata": {
        "id": "_vnlixQaG0-e"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "func_folder = '/content/drive/My Drive/Colab Notebooks'"
      ],
      "metadata": {
        "id": "0xRM9siwIVCi"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sys\n",
        "sys.path.append(func_folder)"
      ],
      "metadata": {
        "id": "gLBwTdWIIURA"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import Contractions\n",
        "from Contractions import *"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G3Ix9QYNIXwi",
        "outputId": "a2e9f842-96cc-4cdd-da41-60f5ee5132e9"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def clean(text):\n",
        "    text = re.sub('[0-9]+.\\t', '', str(text)) # removing paragraph numbers\n",
        "    text = re.sub('U.S.', 'USA', str(text))\n",
        "    text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(' ')])\n",
        "    text = re.sub('\\n ', '', str(text))\n",
        "    text = re.sub('\\n', ' ', str(text))\n",
        "    text = re.sub(\"'s\", '', str(text))\n",
        "    text = re.sub(\"-\", ' ', str(text))\n",
        "    text = re.sub(\"— \", '', str(text))\n",
        "    text = re.sub('\\\"', '', str(text))\n",
        "    text = re.sub(\"Mr\\.\", 'Mr', str(text))\n",
        "    text = re.sub(\"Mrs\\.\", 'Mrs', str(text))\n",
        "    text = re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\", str(text))\n",
        "    text = re.sub(r'(?<=[^\\s0-9])(?=[.,;?])', r' ', text) # add space around punctuation, i.e. treat them as token\n",
        "    text = re.sub(r'\\s\\s', ' ', text)\n",
        "\n",
        "    return text"
      ],
      "metadata": {
        "id": "a9hLS4ypHJVx"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data['text_clean'] = data['text'].apply(clean)"
      ],
      "metadata": {
        "id": "HAYFlDtaIdEC"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "6yYHwp9i5ra_"
      },
      "outputs": [],
      "source": [
        "def sequence_generator(texts,\n",
        "                      training_length, \n",
        "                      result_length, \n",
        "                      max_train=100000,\n",
        "                      start_end_tokens=False,\n",
        "                      lower=True):\n",
        "\n",
        "    tokenizer = Tokenizer(lower=lower)\n",
        "    tokenizer.fit_on_texts(texts)\n",
        "\n",
        "    word_idx = tokenizer.word_index\n",
        "    idx_word = tokenizer.index_word\n",
        "    num_words = len(word_idx) + 1\n",
        "    word_counts = tokenizer.word_counts\n",
        "\n",
        "    print(f'There are {num_words} unique words.')\n",
        "\n",
        "    # import pickle\n",
        "    # with open('tokenizer.pickle', 'wb') as handle:\n",
        "    #     pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
        "\n",
        "    sequences = tokenizer.texts_to_sequences(texts)\n",
        "\n",
        "    # Start-End tokens\n",
        "    # x = word_idx[\"start_token\"]\n",
        "    # y = word_idx[\"end_token\"]\n",
        "\n",
        "    # Limit to sequences with more than training length tokens\n",
        "    seq_lengths = [len(x) for x in sequences]\n",
        "    over_idx = [\n",
        "        i for i, l in enumerate(seq_lengths) if l > (training_length + result_length + 3)]\n",
        "\n",
        "    new_texts = []\n",
        "    new_sequences = []\n",
        "\n",
        "    # Only keep sequences with more than training length tokens\n",
        "    for i in over_idx:\n",
        "        new_texts.append(texts[i])\n",
        "        new_sequences.append(sequences[i])\n",
        "\n",
        "    training_seq = []\n",
        "    labels = []\n",
        "    training_seq_words = []\n",
        "    labels_words = []\n",
        "\n",
        "    for seq in new_sequences:\n",
        "\n",
        "        if len(training_seq) < max_train:\n",
        "            for i in range(training_length, len(seq) - result_length):\n",
        "                # Extract the features and label\n",
        "                extract = seq[i - training_length:i + result_length]\n",
        "                training_seq.append(extract[:training_length])\n",
        "                if start_end_tokens:\n",
        "                    label_adj = [x] + extract[training_length:] + [y]\n",
        "                else: label_adj = extract[training_length:]\n",
        "                labels.append(label_adj)\n",
        "\n",
        "                training_seq_words.append([idx_word[j] for j in extract[:training_length]])\n",
        "                labels_words.append([idx_word[j] for j in extract[training_length:]])\n",
        "\n",
        "    print(f'There are {len(training_seq)} training sequences.')\n",
        "\n",
        "    return word_idx, idx_word, num_words, word_counts, new_texts, new_sequences, training_seq, labels, \\\n",
        "           training_seq_words, labels_words"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "word_idx, idx_word, num_words, word_counts, new_texts, sequences, features, labels, training_seq_words, labels_words = \\\n",
        "    sequence_generator(\n",
        "    data['text_clean'].tolist(), training_length = len_text, result_length = len_result, lower=True)"
      ],
      "metadata": {
        "id": "Tx5GhElBHQ16"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.DataFrame({'features': training_seq_words, 'labels': labels_words})"
      ],
      "metadata": {
        "id": "xKVVoaJzL-NP"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df['labels'] = df['labels'].map(lambda x: x[0])"
      ],
      "metadata": {
        "id": "61dTriZmM2Yg"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "hmGgxfZQNrag",
        "outputId": "2e84b9cc-be28-47e9-bf49-7e3d8a55c651"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                             features         labels\n",
              "0               [by, unanimous, vote]            the\n",
              "1              [unanimous, vote, the]      committee\n",
              "2              [vote, the, committee]       approved\n",
              "3          [the, committee, approved]              a\n",
              "4            [committee, approved, a]          final\n",
              "...                               ...            ...\n",
              "55797     [pressures, and, inflation]   expectations\n",
              "55798  [and, inflation, expectations]            and\n",
              "55799  [inflation, expectations, and]      financial\n",
              "55800  [expectations, and, financial]            and\n",
              "55801           [and, financial, and]  international\n",
              "\n",
              "[55802 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4b0406f8-bbb5-4734-9d9f-2f88249d0f0b\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>features</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[by, unanimous, vote]</td>\n",
              "      <td>the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[unanimous, vote, the]</td>\n",
              "      <td>committee</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[vote, the, committee]</td>\n",
              "      <td>approved</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[the, committee, approved]</td>\n",
              "      <td>a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[committee, approved, a]</td>\n",
              "      <td>final</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55797</th>\n",
              "      <td>[pressures, and, inflation]</td>\n",
              "      <td>expectations</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55798</th>\n",
              "      <td>[and, inflation, expectations]</td>\n",
              "      <td>and</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55799</th>\n",
              "      <td>[inflation, expectations, and]</td>\n",
              "      <td>financial</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55800</th>\n",
              "      <td>[expectations, and, financial]</td>\n",
              "      <td>and</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>55801</th>\n",
              "      <td>[and, financial, and]</td>\n",
              "      <td>international</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>55802 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b0406f8-bbb5-4734-9d9f-2f88249d0f0b')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4b0406f8-bbb5-4734-9d9f-2f88249d0f0b button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4b0406f8-bbb5-4734-9d9f-2f88249d0f0b');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Creating the Transition Matrix\n",
        "\n",
        "The following can be made more Pythonic but it was intuitive to set out each change step by step. First, one-hot encoding is done of the labels to create a dataframe with columns equal to the number of unique vocabulary.\n",
        "\n",
        "Secondly, groupby operation is used on the features phrase (in this case a three word phrase) to group by number of instances of each phrase and a count of the word that follows.\n",
        "\n",
        "Dividing each row by the total of that row creates a probability distribution of the word that follows the three word phrase corresponding to that row. This is our transition matrix."
      ],
      "metadata": {
        "id": "xYTVewii26DR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "markov_matrix = pd.concat([df, pd.get_dummies(df['labels'])], axis=1)"
      ],
      "metadata": {
        "id": "t7Pieoc9OZl9"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "markov_matrix = markov_matrix.drop(['labels'], axis=1)"
      ],
      "metadata": {
        "id": "2vzjG3O1QS7p"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "markov_matrix['features'] = markov_matrix['features'].apply(\" \".join)"
      ],
      "metadata": {
        "id": "4fw_wFgOQ5yl"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "transition_matrix = markov_matrix.groupby('features',as_index=False)[markov_matrix.columns.tolist()].sum()"
      ],
      "metadata": {
        "id": "KBisDV6AU0IR"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# transition_matrix = transition_matrix.div(transition_matrix.sum(axis=1), axis=0)"
      ],
      "metadata": {
        "id": "wdr2glWL9fjA"
      },
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "phrase_dict = transition_matrix['features'].to_dict()\n",
        "word_dict = dict(enumerate(transition_matrix.columns.tolist()))"
      ],
      "metadata": {
        "id": "38004VgCW13k"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "phrase_dict_reversed = {v: k for k, v in phrase_dict.items()}"
      ],
      "metadata": {
        "id": "URmSwpGPbuhS"
      },
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from scipy import sparse"
      ],
      "metadata": {
        "id": "bbbyVnUPLZOq"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pd.DataFrame.sparse.to_coo(transition_matrix)\n",
        "transition_matrix = sparse.csr_matrix(transition_matrix.to_coo())"
      ],
      "metadata": {
        "id": "AiEGjubBIXJJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Generating Words\n",
        "\n",
        "To use our transition matrix, we create dictionaries of the phrases and its index as well as the vocabulary and its index in the transition matrix. Given a phrase, the index of the row is located and the np.random.choice generates the following word weighted by the probabilities in that row. The other function then appends the word to our generated sentence and extracts the last three-word-phrase to loop through the process as many times as defined."
      ],
      "metadata": {
        "id": "pcZpsUZ-3xu9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def add_weights_temperature(input_weights, temperature):\n",
        "    weights = np.where(input_weights == 0, 0, np.log(input_weights + 1e-10)) / temperature\n",
        "    weights = np.exp(weights)\n",
        "    return weights / np.sum(weights)"
      ],
      "metadata": {
        "id": "8myNbiYRWYkJ"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def return_next_word(prefix, temperature=1):\n",
        "    prefix_ind = phrase_dict_reversed[prefix]\n",
        "    weights = transition_matrix.iloc[prefix_ind].values[1:].astype('float64')\n",
        "    prob = weights / sum(weights)\n",
        "    if temperature != 1:\n",
        "        weights = add_weights_temperature(prob, temperature)\n",
        "\n",
        "    token_ind = np.random.choice(range(len(weights)), p=prob)+1\n",
        "    next_word = word_dict[token_ind]\n",
        "    return next_word"
      ],
      "metadata": {
        "id": "ORWw-ufuUQ-p"
      },
      "execution_count": 138,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_words(seed, length):\n",
        "    next_word = return_next_word(seed)\n",
        "    sentence = seed.split()\n",
        "    \n",
        "    for i in range(length):\n",
        "        sentence.append(next_word)\n",
        "        next_word = return_next_word(\" \".join(sentence[-3:]))\n",
        "    \n",
        "    return \" \".join(sentence)"
      ],
      "metadata": {
        "id": "YaQr2AXYUmc-"
      },
      "execution_count": 147,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generate_words(\"accumulated by households\", 100)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "8ciZpuQxekxQ",
        "outputId": "eb3ba3eb-8b53-467b-c9b2-f358439f134e"
      },
      "execution_count": 149,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"accumulated by households since the beginning of the period to negotiations on the debt limit by 480 billion market participants' estimates of the new date when the treasury would exhaust its extraordinary measures and cash balance were wide ranging but some estimates suggested the date might be as early as mid december most market participants anticipated adjustments to the pace of purchases if warranted by changes in the composition of the federal reserve bank of new york until instructed otherwise to execute transactions in the soma in accordance with the committee assessments of maximum employment and inflation and posed considerable risks to the\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 149
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generate_words(\"early as mid\", 100)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "id": "DGwNC9eMiUY0",
        "outputId": "229cc4db-820a-456f-9d40-046a19e64376"
      },
      "execution_count": 150,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'early as mid december most market participants anticipated that the economy were to evolve broadly as they anticipated they judged that the release of pent up demand could boost consumption growth further as social distancing restrictions were imposed to rein in a new wave of covid 19 in the us amid this progress and strong policy support indicators of economic activity and employment had continued to surge and expected that it would likely be appropriate in each subsequent month some participants preferred a somewhat faster pace of reductions that would result in reducing the monthly pace of the recovery indicators of economic activity'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 150
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def create_transition_matrix(self):\n",
        "    row_ind, col_ind, values = [], [], []\n",
        "\n",
        "    for i in range(len(self.tokens[:-self.n])):\n",
        "        ngram = ' '.join(self.tokens[i:i + self.n])\n",
        "        ngram_ind = self.ngram2ind[ngram]\n",
        "        next_word_ind = self.token2ind[self.tokens[i + self.n]]\n",
        "\n",
        "        row_ind.extend([ngram_ind])\n",
        "        col_ind.extend([next_word_ind])\n",
        "        values.extend([1])\n",
        "\n",
        "    S = scipy.sparse.coo_matrix((values, (row_ind, col_ind)), shape=(len(self.ngram2ind), len(self.token2ind)))\n",
        "    return S"
      ],
      "metadata": {
        "id": "bfaozeChGLt7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# LSTM Deep Learning Model\n",
        "\n",
        "Moving on into Deep Learning, we approach this from the perspective of a many-to-one sequence problem. In much of the same way, we generate our features sequences and labels as numpy arrays. We then also one-hot encode the features.\n",
        "\n",
        "The next step is where the earlier-completed tokenization comes into play as we also need to create an embedding for the words. In essence, an embedding is a vector representation of words and their relative distance to other words. There are existing embeddings such as GloVe which have been trained a much larger datasets and can be imported. Otherwise, we can train our own embedding."
      ],
      "metadata": {
        "id": "gt0uYD8n5BX9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "len_text = 30\n",
        "len_result = 1"
      ],
      "metadata": {
        "id": "iffVvv8qwAJw"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "word_idx, idx_word, num_words, word_counts, new_texts, sequences, features, labels, training_seq_words, labels_words = \\\n",
        "    sequence_generator(\n",
        "    data['text_clean'].tolist(), training_length = len_text, result_length = len_result, lower=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OHQNwP5ZwGYk",
        "outputId": "a321fdd1-cddd-42ae-d24e-991870e60257"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "There are 2953 unique words.\n",
            "There are 43934 training sequences.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TRAIN_FRACTION = 0.7\n",
        "RANDOM_STATE = 50"
      ],
      "metadata": {
        "id": "jCgpniopwgrP"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.utils import shuffle"
      ],
      "metadata": {
        "id": "qQvW9vDgwob5"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_train_valid(features,\n",
        "                       labels,\n",
        "                       num_words,\n",
        "                       train_fraction=TRAIN_FRACTION):\n",
        "\n",
        "    features, labels = shuffle(features, labels, random_state=RANDOM_STATE)\n",
        "    train_end = int(train_fraction * len(labels))\n",
        "\n",
        "    train_features = np.array(features[:train_end])\n",
        "    valid_features = np.array(features[train_end:])\n",
        "\n",
        "    train_labels = labels[:train_end]\n",
        "    valid_labels = labels[train_end:]\n",
        "\n",
        "    # Convert to arrays\n",
        "    X_train, X_valid = np.array(train_features), np.array(valid_features)\n",
        "\n",
        "    y_train = np.zeros((len(train_labels), num_words), dtype=np.int8)\n",
        "    y_valid = np.zeros((len(valid_labels), num_words), dtype=np.int8)\n",
        "\n",
        "    # numpy array with one-hot encoding consisting of number of training data \n",
        "    # and size of vocabulary with 1 at the corresponding word following from the features\n",
        "    for example_index, word_index in enumerate(train_labels):\n",
        "        y_train[example_index, word_index] = 1\n",
        "\n",
        "    for example_index, word_index in enumerate(valid_labels):\n",
        "        y_valid[example_index, word_index] = 1\n",
        "\n",
        "    return X_train, X_valid, y_train, y_valid"
      ],
      "metadata": {
        "id": "pLxfDJDO5A0j"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X_train, X_valid, y_train, y_valid = create_train_valid(\n",
        "    features, labels, num_words)"
      ],
      "metadata": {
        "id": "Ut6d4CbSC0Oi"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Pre-trained Embedding with GloVe (Optional)"
      ],
      "metadata": {
        "id": "VaQ9PYFkDBud"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "glove = np.loadtxt('glove.6B.100d.txt', dtype='str', comments=None, encoding=\"utf8\")\n",
        "print(glove.shape)\n",
        "vectors = glove[:, 1:].astype('float')\n",
        "words = glove[:, 0]\n",
        "del glove"
      ],
      "metadata": {
        "id": "IPy2tYLVC8ys"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "word_lookup = {word: vector for word, vector in zip(words, vectors)}\n",
        "embedding_matrix = np.zeros((num_words, vectors.shape[1]))"
      ],
      "metadata": {
        "id": "8gdo1ZSiDETd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "not_found = 0"
      ],
      "metadata": {
        "id": "FMUnlkVoxF_e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i, word in enumerate(word_idx.keys()):\n",
        "    vector = word_lookup.get(word, None)\n",
        "\n",
        "    if vector is not None:\n",
        "        embedding_matrix[i + 1, :] = vector\n",
        "    else:\n",
        "        not_found += 1"
      ],
      "metadata": {
        "id": "0XDIVI4kDHSX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embedding_matrix = embedding_matrix / \\\n",
        "    np.linalg.norm(embedding_matrix, axis=1).reshape((-1, 1))\n",
        "embedding_matrix = np.nan_to_num(embedding_matrix)"
      ],
      "metadata": {
        "id": "o-uuWFgjDilN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def find_closest(query, embedding_matrix, word_idx, idx_word, n=10):\n",
        "\n",
        "    idx = word_idx.get(query, None)\n",
        "    if idx is None:\n",
        "        print(f'{query} not found in vocab.')\n",
        "        return\n",
        "    else:\n",
        "        vec = embedding_matrix[idx]\n",
        "        if np.all(vec == 0):\n",
        "            print(f'{query} has no pre-trained embedding.')\n",
        "            return\n",
        "        else:\n",
        "            # Calculate distance between vector and all others\n",
        "            dists = np.dot(embedding_matrix, vec)\n",
        "\n",
        "            idxs = np.argsort(dists)[::-1][:n]\n",
        "            sorted_dists = dists[idxs]\n",
        "            closest = [idx_word[i] for i in idxs]\n",
        "\n",
        "    print(f'Query: {query}\\n')\n",
        "    max_len = max([len(i) for i in closest])\n",
        "    for word, dist in zip(closest, sorted_dists):\n",
        "        print(f'Word: {word:15} Cosine Similarity: {round(dist, 4)}')"
      ],
      "metadata": {
        "id": "P-eLo6miDmPd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Building and Training LSTM Model\n",
        "\n",
        "Whether or not we have the pre-trained embeddings, we approach this problem based on the sequence of words, which is however long we specify and earch further represented by the dimensions of the embedding.\n",
        "\n",
        "We can train a recurrent neural network with one or several layers on these token sequences with embeddings to predict the label, i.e. the token/word that follows."
      ],
      "metadata": {
        "id": "u2bihDfbw9cu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "LSTM_CELLS = 64\n",
        "embedding_dim = 200\n",
        "EPOCHS = 20\n",
        "BATCH_SIZE = 2048\n",
        "VERBOSE = 0"
      ],
      "metadata": {
        "id": "hd1WTCZpzGB6"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")"
      ],
      "metadata": {
        "id": "0Wzh2ODX7yxy"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.models import Sequential, load_model\n",
        "from keras.layers import LSTM, Dense, Dropout, Embedding, Masking, Bidirectional"
      ],
      "metadata": {
        "id": "W1VpSK4BDoBh"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = Sequential()"
      ],
      "metadata": {
        "id": "TAYUg4IPDpoi"
      },
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.add(Embedding(input_dim=num_words, output_dim=embedding_dim, trainable=True))"
      ],
      "metadata": {
        "id": "cComd8_dxmSL"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.add(LSTM(LSTM_CELLS, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))"
      ],
      "metadata": {
        "id": "qPK31jBOxsL5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.add(LSTM(LSTM_CELLS, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))"
      ],
      "metadata": {
        "id": "yxdRzWF9z7Zy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.add(Bidirectional(LSTM(LSTM_CELLS, return_sequences=False, dropout=0.1, recurrent_dropout=0.1)))"
      ],
      "metadata": {
        "id": "4PEiD8f_zJEH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.add(Dense(num_words, activation='softmax'))"
      ],
      "metadata": {
        "id": "IwnHqhk_zc9u"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])"
      ],
      "metadata": {
        "id": "MPImTde2zhiN"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "72ZDoC3wzklg",
        "outputId": "3d86d2c3-1bd2-4e6c-e3d3-688f6a6f8aa5"
      },
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding_1 (Embedding)     (None, None, 200)         590600    \n",
            "                                                                 \n",
            " lstm_3 (LSTM)               (None, None, 64)          67840     \n",
            "                                                                 \n",
            " lstm_4 (LSTM)               (None, None, 64)          33024     \n",
            "                                                                 \n",
            " bidirectional_1 (Bidirectio  (None, 128)              66048     \n",
            " nal)                                                            \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 2953)              380937    \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,138,449\n",
            "Trainable params: 1,138,449\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# checkpoint_path = \"training_1/cp.ckpt\"\n",
        "# checkpoint_dir = os.path.dirname(checkpoint_path)\n",
        "# cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,\n",
        "                                                 save_weights_only=True,\n",
        "                                                 verbose=1)"
      ],
      "metadata": {
        "id": "7waQ2a-W_DmM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "history = model.fit(\n",
        "    X_train,\n",
        "    y_train,\n",
        "    epochs=EPOCHS,\n",
        "    batch_size=BATCH_SIZE,\n",
        "    verbose=VERBOSE,\n",
        "    validation_data=(X_valid, y_valid))"
      ],
      "metadata": {
        "id": "KyWv3rNNz6kP"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib import pyplot\n",
        "\n",
        "pyplot.plot(history.history['loss'], label='train')\n",
        "pyplot.plot(history.history['val_loss'], label='test')\n",
        "pyplot.legend()\n",
        "pyplot.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "RDzU5n335HEx",
        "outputId": "29ec6381-3c89-43eb-a6fb-8b0ac439b76e"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3RcZ3nv8e8zF2kkWZYsWb7IduJLgokpxDFuLiXkQnCIQ0nglJMTWlpKYTnpARactcghrJa00LW66OEcVqEUfAINtAdIgdCUtHWCEwiXVQhgO05iHCd2TMDyPXYs3yRrLs/5Y++RRqMZaWRJM9Ke32etWXvv9333zDNbo2f2vPvde5u7IyIi0RWrdQAiIjK1lOhFRCJOiV5EJOKU6EVEIk6JXkQk4hK1DqCUuXPn+tKlS2sdhojIjLF169aX3L2rVN20TPRLly5ly5YttQ5DRGTGMLNfl6tT142ISMQp0YuIRJwSvYhIxE3LPnoRkfFKp9P09PTQ399f61CmVCqVYvHixSSTyYrXUaIXkUjo6emhtbWVpUuXYma1DmdKuDvHjh2jp6eHZcuWVbyeum5EJBL6+/vp7OyMbJIHMDM6OzvH/atFiV5EIiPKST7vfN5jZBJ9Lud87vu7+dHzR2sdiojItBKZRB+LGff+aC/fe/ZwrUMRkTp04sQJPv/5z497vZtvvpkTJ05MQURDIpPoAbrbm9h/ItpH3EVkeiqX6DOZzKjrbdq0ifb29qkKC4jYqJtF7U0cONFX6zBEpA7dfffdvPDCC6xevZpkMkkqlWLOnDns2rWL559/nre+9a3s27eP/v5+PvjBD7JhwwZg6JIvp0+fZv369Vx99dX85Cc/YdGiRXznO9+hqalpwrFFKtF3tzex9Tcv1zoMEamxj//bL9l54OSkPueq7tn8xVteVbb+k5/8JDt27GD79u384Ac/4M1vfjM7duwYHAZ533330dHRQV9fH7/927/N7/3e79HZ2TnsOXbv3s3999/PF7/4RW677Ta+/e1v8853vnPCsUeu6+bE2TRnzo3+U0lEZKpdfvnlw8a6f/azn+XSSy/lyiuvZN++fezevXvEOsuWLWP16tUAvPa1r+XFF1+clFgitkefAuBgbx8XzWutcTQiUiuj7XlXS0tLy+D8D37wAx577DF++tOf0tzczHXXXVdyLHxjY+PgfDwep69vcrqiI7VHv6g96MvSAVkRqbbW1lZOnTpVsq63t5c5c+bQ3NzMrl27eOKJJ6oaW8T26INErwOyIlJtnZ2dvO51r+O3fuu3aGpqYv78+YN1N910Exs3buSSSy5h5cqVXHnllVWNLVKJfl5rI/GYKdGLSE18/etfL1ne2NjIww8/XLIu3w8/d+5cduzYMVj+4Q9/eNLiilTXTSIeY8HsFPuV6EVEBo2Z6M1spZltL3icNLMPFbW5q6B+h5llzawjrHvRzJ4J66b8/oAL21LaoxcRKTBm1427PwesBjCzOLAfeLCozaeAT4Vt3gL8D3c/XtDkend/abKCHk13exPb903t6cQiIjPJeLtubgBecPeyN6EF3gHcf/4hTUx3exMHe/vI5bxWIYiITCvjTfS3M0oSN7Nm4Cbg2wXFDmw2s61mtmGUdTeY2RYz23L06PlfgXJRe4p01nnp9Lnzfg4RkSipONGbWQNwC/CtUZq9BfjPom6bq919DbAeeJ+ZXVNqRXe/193Xuvvarq6uSsMaoXtwLL366UVEYHx79OuBbe4+2nWAR+zxu/v+cHqEoG//8vEGOR5DY+l10pSIVM/5XqYY4G//9m85e/bsJEc0ZDyJftS+dzNrA64FvlNQ1mJmrfl54EZgR+lnmBw6aUpEamE6J/qKTpgKk/Q64I6CsjsB3H1jWPQ2YLO7nylYdT7wYHjrqwTwdXd/ZBLiLmt2KsGsxoS6bkSkqgovU7xu3TrmzZvHN7/5Tc6dO8fb3vY2Pv7xj3PmzBluu+02enp6yGazfOxjH+Pw4cMcOHCA66+/nrlz5/L4449PemwVJfoweXcWlW0sWv4K8JWisr3ApROKcJzMjO52jaUXqWsP3w2Hnpnc51zwalj/ybLVhZcp3rx5Mw888AA///nPcXduueUWfvSjH3H06FG6u7v5j//4DyC4Bk5bWxuf/vSnefzxx5k7d+7kxhyK1Jmxed3tTRzoVaIXkdrYvHkzmzdv5rLLLmPNmjXs2rWL3bt38+pXv5pHH32Uj3zkI/z4xz+mra2tKvFE6lo3ed3tTTzd01vrMESkVkbZ864Gd+ejH/0od9xxx4i6bdu2sWnTJv78z/+cG264gXvuuWfK44nkHv2i9iaOnxmgP52tdSgiUicKL1P8pje9ifvuu4/Tp08DsH//fo4cOcKBAwdobm7mne98J3fddRfbtm0bse5UiOgefXADkgMn+ljeNavG0YhIPSi8TPH69ev5/d//fa666ioAZs2axVe/+lX27NnDXXfdRSwWI5lM8oUvfAGADRs2cNNNN9Hd3T0lB2PNffpdKmDt2rW+Zcv5X//sZ3uP8d/ufYKvvucKrr54ag5uiMj08uyzz3LJJZfUOoyqKPVezWyru68t1T6SXTcaSy8iMiSSiX5BWwozXQZBRAQimuiT8RjzWzWWXqTeTMeu6Ml2Pu8xkokeggOyGksvUj9SqRTHjh2LdLJ3d44dO0YqlRrXepEcdQNBP/0vD5ysdRgiUiWLFy+mp6eHiVzmfCZIpVIsXrx4XOtENtEvam9i887DuDvhtXZEJMKSySTLli2rdRjTUoS7bpoYyOQ4dmag1qGIiNRUpBM9aIiliEiEE/3Q2bEiIvUssol+0eAtBXWnKRGpb5FN9G1NSZob4tqjF5G6F9lEb2YsbNNJUyIikU30EN6ARIleROpcpBP9ovYm9dGLSN0bM9Gb2Uoz217wOGlmHypqc52Z9Ra0uaeg7iYze87M9pjZ3VPxJsrpbm/ipdPndAMSEalrY54Z6+7PAasBzCwO7AceLNH0x+7+u4UFYfu/B9YBPcAvzOwhd9850cArkR9Lf6i3n6VzW6rxkiIi0854u25uAF5w919X2P5yYI+773X3AeCfgVvH+ZrnTWPpRUTGn+hvB+4vU3eVmT1lZg+b2avCskXAvoI2PWHZCGa2wcy2mNmWyboo0dBYeiV6EalfFSd6M2sAbgG+VaJ6G3Chu18K/B3wr+MNxN3vdfe17r62q6trvKuXtKAtv0evA7IiUr/Gs0e/Htjm7oeLK9z9pLufDuc3AUkzm0vQn7+koOnisKwqGhNxulob1XUjInVtPIn+HZTptjGzBRZeC9jMLg+f9xjwC+BiM1sW/iK4HXhoYiGPT3d7k25AIiJ1raLr0ZtZC8HImTsKyu4EcPeNwNuBPzWzDNAH3O7BbV4yZvZ+4LtAHLjP3X85uW9hdIvaU+w6dKqaLykiMq1UlOjd/QzQWVS2sWD+c8Dnyqy7Cdg0gRgnpLutie/vOqIbkIhI3Yr0mbEQdN30p3O8fDZd61BERGqiLhI9aCy9iNSvyCd6jaUXkXoX+USvs2NFpN5FPtF3tDTQmIhxsFcnTYlIfYp8ojez8HLF2qMXkfoU+UQPugGJiNS3Okn0uqWgiNSvOkn0TRw5dY6BTK7WoYiIVF3dJHp3OHxSB2RFpP7URaLXWHoRqWd1keh1dqyI1LO6SPQL23TSlIjUr7pI9KlknM6WBvbrTlMiUofqItGDxtKLSP2qo0SvsfQiUp/qKNEHe/TBja9EROpH3ST6Re1NnBnIcrIvU+tQRESqqm4SfbfG0otInRoz0ZvZSjPbXvA4aWYfKmrzB2b2tJk9Y2Y/MbNLC+peDMu3m9mWqXgTldBYehGpV2PeHNzdnwNWA5hZHNgPPFjU7FfAte7+spmtB+4Friiov97dX5qckM/P4A1IepXoRaS+jJnoi9wAvODuvy4sdPefFCw+ASyeaGCTbW5LIw3xmLpuRKTujLeP/nbg/jHavAd4uGDZgc1mttXMNpRbycw2mNkWM9ty9OjRcYY1tljMWNie4oBOmhKROlNxojezBuAW4FujtLmeINF/pKD4andfA6wH3mdm15Ra193vdfe17r62q6ur0rCG5HJwaAe8/GLZJt1tOmlKROrPePbo1wPb3P1wqUozew3wJeBWdz+WL3f3/eH0CEHf/uXnH+4YvvgG+MWXylbr7FgRqUfjSfTvoEy3jZldAPwL8Ifu/nxBeYuZtebngRuBHecf7ihiMehYDsf2lm2yqD3F4ZP9pLO6AYmI1I+KDsaGSXodcEdB2Z0A7r4RuAfoBD5vZgAZd18LzAceDMsSwNfd/ZHJfAPDdK6Al3aXre5ubyIX3oBk8ZzmKQtDRGQ6qSjRu/sZgkReWLaxYP69wHtLrLcXuLS4fMp0LIfdmyGXhVh8RPXQWHolehGpH9E6M7ZzBWQHoLenZLVOmhKRehSxRH9RMD3+Qsnq/ElTGksvIvUkWom+Y0UwPVY60Tc3JJjTnNQevYjUlWgl+tYFkGwpm+hBQyxFpP5EK9GbBQdky3TdQD7R6+xYEakf0Ur0EByQHWWPfpH26EWkzkQz0b/8ImTTJau721OcOpfhZH/pehGRqIleou9YAZ6FE78pWZ0fYnlQ3TciUieil+g7Rx95o7H0IlJvIpjoxxhL36ZbCopIfYleom/uhMY2OLanZHVXayOJmGmPXkTqRvQSvRl0Li/bdROPGQvaUkr0IlI3opfoIei+0Vh6EREgqom+YwWc2Afp0sl8UXuT+uhFpG5EM9F3rgC87G0Fu9tTHDrZTzbnVQ1LRKQWopno8xc3K3sVyyayOefIKXXfiEj0RTPRdy4PphpLLyIS0UTfNCcYZllmiOWi9vxYeu3Ri0j0RTPRQ9B9c7z0jcIXtgU3INEevYjUgzETvZmtNLPtBY+TZvahojZmZp81sz1m9rSZrSmoe5eZ7Q4f75qKN1HSKFexbE0lmZ1KKNGLSF0Y8+bg7v4csBrAzOLAfuDBombrgYvDxxXAF4ArzKwD+AtgLeDAVjN7yN1fnrR3UE7nCnjqfhg4Aw0tI6p1AxIRqRfj7bq5AXjB3X9dVH4r8E8eeAJoN7OFwJuAR939eJjcHwVumnDUlRgceVO6+yYYS68+ehGJvvEm+tuB+0uULwL2FSz3hGXlykcwsw1mtsXMthw9enScYZVQwVUstUcvIvWg4kRvZg3ALcC3piIQd7/X3de6+9qurq6JP2FHOMRylLH0vX1pTp/LTPy1RESmsfHs0a8Htrn74RJ1+4ElBcuLw7Jy5VOvsRVmLRhljz4YeXNQe/UiEnHjSfTvoHS3DcBDwB+Fo2+uBHrd/SDwXeBGM5tjZnOAG8Oy6hhl5M3QWHolehGJtjFH3QCYWQuwDrijoOxOAHffCGwCbgb2AGeBd4d1x83sr4BfhKt9wt2PT1r0Y+lYDs8/UrJq6OxYHZAVkWirKNG7+xmgs6hsY8G8A+8rs+59wH0TiPH8dV4EZ45Cfy+k2oZVzWttJK4bkIhIHYjumbEw6sibRDzGgtm6AYmIRF+0E/0YY+m721PqoxeRyIt4ol8WTEcbS9+rRC8i0RbtRJ9sgrYlo46lP9SrG5CISLRFO9FDMPKmzOWKu9ubSGedl06fq3JQIiLVE/1EP+pY+uCkKfXTi0iURT/Rd6yA/hNwduTw/YVtutOUiERf9BN950XBtMRevW4pKCL1oA4SfX4s/ch++tmpBLMaEzo7VkQiLfqJvv1CsFjJkTdmRne7TpoSkWiLfqJPNED7BRpLLyJ1K/qJHoJ++lGGWKrrRkSirD4SfceK4DIIPvLEqEXtTRw/M0DfQLYGgYmITL36SPSdK2DgNJw+MqIqfwMSdd+ISFTVT6KHkgdkuzWWXkQirj4SfUf5IZYaSy8iUVcfib5tCcSSJUfeLGhLYQb7dUBWRCKqPhJ9PAFzlpbsuknGY8xv1Vh6EYmu+kj0EA6xLH8DEiV6EYmqihK9mbWb2QNmtsvMnjWzq4rq7zKz7eFjh5llzawjrHvRzJ4J67ZMxZuoSOeKYI8+lxtRFYylV6IXkWiqdI/+M8Aj7v5K4FLg2cJKd/+Uu69299XAR4Efunvh5SKvD+vXTkrU56NjOWT64dSBEVWL2ps40NtPTjcgEZEIGjPRm1kbcA3wDwDuPuDuJ0ZZ5R3A/ZMT3iQa5Ubh3e1NDGRyHDszUOWgRESmXiV79MuAo8CXzexJM/uSmbWUamhmzcBNwLcLih3YbGZbzWxDuRcxsw1mtsXMthw9enQcb6FCg5cr1hBLEakvlST6BLAG+IK7XwacAe4u0/YtwH8Wddtc7e5rgPXA+8zsmlIruvu97r7W3dd2dXVV/g4q1doNiVRwKYQig2fHKtGLSARVkuh7gB53/1m4/ABB4i/ldoq6bdx9fzg9AjwIXH5+oU5QLBbeP3Zk182icI9etxQUkSgaM9G7+yFgn5mtDItuAHYWtwv78q8FvlNQ1mJmrfl54EZgxyTEfX46lpccS9/WlKS5Ia6rWIpIJCUqbPcB4Gtm1gDsBd5tZncCuPvGsM3bgM3ufqZgvfnAg2aWf62vu/sjkxL5+ei8CJ7/LmQzwUlUoeAGJBpiKSLRVFGid/ftQPHQyI1Fbb4CfKWobC/BcMzpoXMF5NLQuw86lg2r0g1IRCSq6ufMWBi6uFmJ7ptFOjtWRCKqvhL94BDLEiNv2pp46fQA/WndgEREoqW+Ev2sedAwa9Sx9Ad7dUBWRKKlvhK9WdmRNws1ll5EIqq+Ej0EB2Q1ll5E6kgdJvqL4MRvIDP8ujYL2oI9+v0vK9GLSLTUX6LvWAGehRO/HlbcmIizfG4LT+4b7XptIiIzT/0l+lGuYnnDJfP46Qsvcao/XeWgRESmTv0l+lHG0q9btYB01vnh81Nw9UwRkRqpv0Tf3AGp9pJDLF974Rw6Whp4dOfhGgQmIjI16i/Rm5UdeROPGW945Twe33WEdHbkLQdFRGai+kv0EHTflLguPcC6VfM52Z/h5786XrJeRGSmqc9E37kCensgPXIo5esvnktjIqbuGxGJjDpN9BcBDsd/NaKquSHB6y+ey6M7D+Oum4WLyMxXn4m+Y3kwLTHyBoLum/0n+th58GQVgxIRmRr1mehHGUsP8IZXzscMdd+ISCTUZ6JPtUFLV9k9+q7WRtZcMEeJXkQioT4TPQQjb8rs0QPcuGo+vzxwUhc5E5EZr6JEb2btZvaAme0ys2fN7Kqi+uvMrNfMtoePewrqbjKz58xsj5ndPdlv4LyVGUuft27VfAAe0169iMxwle7RfwZ4xN1fSXAP2GdLtPmxu68OH58AMLM48PfAemAV8A4zWzUJcU9cx3I4fQjOnS5ZvbxrFiu6WtR9IyIz3piJ3szagGuAfwBw9wF3r/QSj5cDe9x9r7sPAP8M3Hq+wU6q/G0Fy/TTQ3Dtmyf2HqO3Txc5E5GZq5I9+mXAUeDLZvakmX3JzFpKtLvKzJ4ys4fN7FVh2SJgX0GbnrCs9sYYeQNB900m5/zguSNVCkpEZPJVkugTwBrgC+5+GXAGKO5r3wZc6O6XAn8H/Ot4AzGzDWa2xcy2HD1ahatHjjGWHuCyJe3MndXIZnXfiMgMVkmi7wF63P1n4fIDBIl/kLufdPfT4fwmIGlmc4H9wJKCpovDshHc/V53X+vua7u6usb5Ns5DQwu0LoRjpa95AxCLGW+8ZB4/fO4o5zLZqY9JRGQKjJno3f0QsM/MVoZFNwA7C9uY2QIzs3D+8vB5jwG/AC42s2Vm1gDcDjw0ifFPTOdFJS9XXGjdqvmcPpfhib26yJmIzEyJCtt9APhamKz3Au82szsB3H0j8HbgT80sA/QBt3twoZiMmb0f+C4QB+5z919O9ps4bx3LYde/j9rkdRfNpSkZ59Gdh7j2FVX4pSEiMskqSvTuvh1YW1S8saD+c8Dnyqy7Cdh0vgFOqc4VcPYY9J2ApvaSTVLJONe8Yi6P7TzCX93qhD9cRERmjPo9MxZGva1goXWrFnDoZD/P7O+tQlAiIpOrvhN9fiz9KEMsAd7wynnEdJEzEZmh6jvRz1kK2JiJvqOlgbVLO5ToRWRGqu9En0xB25Ixu24guMjZrkOn2Hf8bBUCExGZPPWd6GHMi5vl5S9yppOnRGSmUaLPJ/oxbht4YWcLr5g/i0d3HqpSYCIik0OJvmMFnOsNhlmOYd2q+fzixZc5cXagCoGJiEwOJfoKLm6Wt27VArI55/u7dJEzEZk5lOgHh1iOfikEgNcsamNea6NG34jIjKJE334BWLyikTexmPHGVfP54fNH6U/rImciMjMo0ceTMOfCirpuIOinPzuQ5acvjN2nLyIyHSjRQ3BAtoI9eoDfWdFJS0NcwyxFZMZQoofwcsV7xxxiCdCYiHPdynk89uxhcrmx24uI1JoSPQQjb9Jn4FRlY+TXrZrP0VPn2N5T6a1zRURqR4keKrqtYKHrV84jHjONvhGRGUGJHsY1lh6grTnJFct0kTMRmRmU6CG4sFm8oaKx9HnrVs1nz5HT/OqlM1MYmIjIxCnRA8TiMGcZHC9/o/Bi+Yuc6do3IjLdKdHnVXgVy7zFc5q5ZOFsdd+IyLRXUaI3s3Yze8DMdpnZs2Z2VVH9H5jZ02b2jJn9xMwuLah7MSzfbmZbJvsNTJqO5cEefS5X8SrrVs1n669f5tjpc1MYmIjIxFR0c3DgM8Aj7v52M2sAmovqfwVc6+4vm9l64F7gioL66939pYmHO4U6L4LsOTjZE1wWoQI3rprPZ7+3m+/tOsJta5dMcYAiMiPkcuBZyGWLpjnwUnW5oWWLQdfKSQ9pzERvZm3ANcAfA7j7ADDsOr3u/pOCxSeAxZMXYpUUjrypMNG/qns23W0pHt15WIle6ktmAPpeDh/Hg+nZ45DuA8ITCQdPQCxcHq0uXPZcsOy5cNmH1h1Rlytdl0+g+cTqueGJNl83rF0OcpngkU2PMp8O1svPZ8O6XNhuIlrmwV27J/YcJVSyR78MOAp8OeyS2Qp80N3LDTd5D/BwwbIDm83Mgf/r7vdOJOAp0xEm+oNPwZIrINkEZqOuYhZc5OybW/bRN5ClqSFehUClqvJJpjDpMEpZcULIhglgcDldpi5cx3PBXt3gw4qWSz1s+PxEpPuGknZhAs8n9LNhch84NfFte95s6D2PmI8FgyvMgosV5rdLLD8f1sUK6vLtYuF8PAmxRPBINoXzSYgnyswng+eL5deLB8+Tf77B5eJ4StQliztLJkcliT4BrAE+4O4/M7PPAHcDHytuaGbXEyT6qwuKr3b3/WY2D3jUzHa5+49KrLsB2ABwwQWV7VFPqtaF0DgbHvuL4BFvhKY5JR7tw5b/65wYWzM9bHlqO69/zSugYdbE/9mixj3YAypOavm9oMKElx2ATH/wSIfTzDnI9AXTdDjNt8nX58uzA0PJdtgjW/Ca2eF7YPnlbDrc2yvY86xnFoNU+Hlv7oBZ86HrkmC+8H+iuQOawrKGFgaT77Dnyi/b8PniuvzrFn6BFSZz/W+dF/Mxru9iZguAJ9x9abj8euBud39zUbvXAA8C6939+TLP9ZfAaXf/36O95tq1a33Llhoct92/DQ49XfCTNP84MXw5PcoNwmOJ8B+go+CfoPifokR9snnkhziXCxJcuj94zUw4HW05O1DwM7Wo/6+S8sEkmBlKzsOWyyXPgvalkvlkSzRBohESqeAm74lUsBxvHNqriiUK9s7iQ3tgo9Xn9/gGE1JhgrEwFxWXFUzze4LxZMHeXtFy2bpE8Lxe1A1R9lGmfiLJMJEa+qym2oO9UpkRzGyru68tVTfmHr27HzKzfWa20t2fA24Adha9wAXAvwB/WJjkzawFiLn7qXD+RuATE3gvU2vRmuAxlnQ/9A8l/3u/u5UDBw9wzw0LifUX9Vme2Bd0B431BZH/BWEW7J2m+4KDwxNV+PO08OdrrFx5fChZ5RNgyZ+yiRIJND48cQ2uWyapDfvpG9bFG4OEnSxI5PlHPqHHG7RnJzIOlY66+QDwtXDEzV7g3WZ2J4C7bwTuATqBz1vwD5gJv1nmAw+GZQng6+7+yOS+hRpIpiC5AFoXALDgiqX89f1P8ruLr2Lt0o7y66X7Sx/AGuz/PB4+f3P4Gs1hgitebhp6JArn80mwsE9SCVGk3lWU6N19O1D8k2BjQf17gfeWWG8vcGlxedRct7KLZDy4yNmoiT6ZguRCmL2wesGJSN1TB9wkmJ1KcuXyTjbvPMxYxzxERKpNiX6SrFs1n1+9dIYXjp6udSgiIsMo0U+SN14SXOTszx7cwZf/81ds33eCgUzll1MQEZkqlR6MlTF0tzfxgTdcxDe37OPj/xYMSmqIx3jVotmsXtLOZRfM4bIl7Sye04TpAKmIVNGY4+hroWbj6CfJwd4+nvzNCbbvO8H235zg6f0n6E8He/dzZzUMJv7VS9p5zeI2WlPJGkcsIjPdhMbRy/gtbGti4aubuPnVweiadDbHc4dO8WSY+J/c9zKPPXsECEY/XjxvFpctmcPqC9q5oKOZ2akks5sSzE4laU0lSMTVwyYi50979DXSezbN9p4g8W/f9zJP7jvBibOlzyBtaYgzuyk57AsgWE6MKE81xGmIx2hIxEjGYyTjRuPg/FB5vk08pm4kkSjQHv001Nac5NpXdHHtK7oAcHd+c/wsh3r7Odmf4WRfmpP9aU72ZcLp0PKhk/08f+TUYN1EvqtjxojEH48ZMTNiMYibEQuX8/PxGEG9BW3jZpgx+KXhDjl3PHxf+eWcM6IsP4VgGjMjGY+RiBvJWIxkwkjEgi+soDycj4VtwuVE+EVm4fNkc07WnVzOyebyrx+UD00J6gvKjaH3MvTeGdwmg9vGwu1QtG0S4fZLxo14GHc8FsQbj9lgzMPahOsk4jFiRrCdwu2Vj7twGwbLTi5XsF3DacwglYzTmIjRmIyTSsZoTATTfHkqGScRMx0rqiNK9NOEmXFhZwsXdraMa71czjkzkOFkf4bes2n6M1nSmRzprDOQzTKQcdLZHAOZHOls8DgX1ueXBzI5BsLpUCJkWGIcTJ65IKkUJsdcDrLug6OMYhZcEyZmYBYLrrYQJvANzrYAAAenSURBVFCzIHFa2C5fZgTdWDmHTHYovv50jkw2w0DWyWRzZHLB62RyOTJZZyAbTDO5YB3C5xn8UhpMzgwuF5YXJ/X8Ns2/P3eGfWkUfkkUb5vc9PtxXFb+C6Ew+TcmYjQmCn71FUwbwp2BZMJoiMdJJozGol+JyUSMhvDLKxEb/sWXn8YHl2MjyhMxI5WMD3ZZ5v8eMnFK9DNcLGa0ppK0ppIsam+qdTg1le+GrNWean6vOp0NvjDzX0CZnA+WpbMeTsM2ufwXVVDm5H8tDX0R5n9BxAq+KPNlg/Xhr6xszulPZzmXydGfztKfznEuk+VcOO1P54bVD7bL5DgXLue//M+cywwtZ3OkMx5Oc5wL20wVM2htTNDWnKQt7J5saxp6zA4fw8pSCVpTSZob4jQl4/qiKKBEL5FR664IMyNuEI/Vx30J3Ie+oPK/CtPZ4JdPJudkwy+5bPjIz2eywa+gwTbZoa62swPZoJuyL01v+DjZn6G3L83uI6cHyyr5kkklY7Q0JGhqiAfJvyFBczI/H0ybGxLhNKhPJWPhr4wY8RiDv07iw36dxIp+ndhgt1wirE8mgi65wa7G+NDz1OJzqkQvIufFzAaTWHNDdV+7P50d9mWQf5w+l+HsQJazA1n6BjLhNFg+M5ChbyDLwd40feksZwvqM1Xsd2sIj0ElYkZDIkYiPN7UEI8xt7WRb95x1dhPMk5K9CIy4+SPL8ybnZqU5xvI5OgbyNKfyRb9AsmRzUEmlxvxqyTf9ZbzoeV0LjyWNHj8KDyulD+WlM0NP96Ub5MN1m2ZorvUKdGLSN1rCA9CtxHNkxd1Jo6ISMQp0YuIRJwSvYhIxCnRi4hEnBK9iEjEKdGLiEScEr2ISMQp0YuIRNy0vB69mR0Ffn2eq88FXprEcCab4psYxTcxim9ipnN8F7p7V6mKaZnoJ8LMtpS7+P50oPgmRvFNjOKbmOkeXznquhERiTglehGRiItior+31gGMQfFNjOKbGMU3MdM9vpIi10cvIiLDRXGPXkRECijRi4hE3IxN9GZ2k5k9Z2Z7zOzuEvWNZvaNsP5nZra0irEtMbPHzWynmf3SzD5Yos11ZtZrZtvDxz3Vii98/RfN7JnwtbeUqDcz+2y4/Z42szVVjG1lwXbZbmYnzexDRW2quv3M7D4zO2JmOwrKOszsUTPbHU7nlFn3XWGb3Wb2rirG9ykz2xX+/R40s/Yy6476WZjC+P7SzPYX/A1vLrPuqP/rUxjfNwpie9HMtpdZd8q334S5+4x7AHHgBWA50AA8BawqavPfgY3h/O3AN6oY30JgTTjfCjxfIr7rgH+v4TZ8EZg7Sv3NwMOAAVcCP6vh3/oQwckgNdt+wDXAGmBHQdn/Au4O5+8G/qbEeh3A3nA6J5yfU6X4bgQS4fzflIqvks/CFMb3l8CHK/j7j/q/PlXxFdX/H+CeWm2/iT5m6h795cAed9/r7gPAPwO3FrW5FfjHcP4B4Aar0u3X3f2gu28L508BzwKLqvHak+hW4J888ATQbmYLaxDHDcAL7n6+Z0pPCnf/EXC8qLjwM/aPwFtLrPom4FF3P+7uLwOPAjdVIz533+zumXDxCWDxZL9upcpsv0pU8r8+YaPFF+aN24D7J/t1q2WmJvpFwL6C5R5GJtLBNuGHvRforEp0BcIuo8uAn5WovsrMnjKzh83sVVUNDBzYbGZbzWxDifpKtnE13E75f7Babj+A+e5+MJw/BMwv0Wa6bMc/IfiFVspYn4Wp9P6wa+m+Ml1f02H7vR447O67y9TXcvtVZKYm+hnBzGYB3wY+5O4ni6q3EXRHXAr8HfCvVQ7vandfA6wH3mdm11T59cdkZg3ALcC3SlTXevsN48Fv+Gk5VtnM/gzIAF8r06RWn4UvACuA1cBBgu6R6egdjL43P+3/l2Zqot8PLClYXhyWlWxjZgmgDThWleiC10wSJPmvufu/FNe7+0l3Px3ObwKSZja3WvG5+/5wegR4kOAncqFKtvFUWw9sc/fDxRW13n6hw/nurHB6pESbmm5HM/tj4HeBPwi/jEao4LMwJdz9sLtn3T0HfLHM69Z6+yWA/wJ8o1ybWm2/8Zipif4XwMVmtizc67sdeKiozUNAfoTD24Hvl/ugT7awT+8fgGfd/dNl2izIHzMws8sJ/hZV+SIysxYza83PExy021HU7CHgj8LRN1cCvQXdFNVSdk+qltuvQOFn7F3Ad0q0+S5wo5nNCbsmbgzLppyZ3QT8T+AWdz9bpk0ln4Wpiq/wmM/byrxuJf/rU+mNwC537ylVWcvtNy61Php8vg+CUSHPExyR/7Ow7BMEH2qAFMFP/j3Az4HlVYztaoKf8U8D28PHzcCdwJ1hm/cDvyQYRfAE8DtVjG95+LpPhTHkt19hfAb8fbh9nwHWVvnv20KQuNsKymq2/Qi+cA4CaYJ+4vcQHPP5HrAbeAzoCNuuBb5UsO6fhJ/DPcC7qxjfHoL+7fxnMD8KrRvYNNpnoUrx/b/ws/U0QfJeWBxfuDzif70a8YXlX8l/5graVn37TfShSyCIiETcTO26ERGRCinRi4hEnBK9iEjEKdGLiEScEr2ISMQp0YuIRJwSvYhIxP1/wj8c3wlrIpYAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.save(root_folder+'/BiLSTMNLG', save_format='tf')"
      ],
      "metadata": {
        "id": "nEM7K1zq5VLU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Load trained model...**"
      ],
      "metadata": {
        "id": "535TesKaIiqw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import tensorflow as tf"
      ],
      "metadata": {
        "id": "7-nTTnsk-xJ9"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "load_model = Sequential()\n",
        "load_model.add(Embedding(input_dim=num_words, output_dim=embedding_dim, trainable=True))\n",
        "load_model.add(LSTM(LSTM_CELLS, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))\n",
        "load_model.add(LSTM(LSTM_CELLS, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))\n",
        "load_model.add(Bidirectional(LSTM(LSTM_CELLS, return_sequences=False, dropout=0.1, recurrent_dropout=0.1)))\n",
        "load_model.add(Dense(num_words, activation='softmax'))\n",
        "load_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])"
      ],
      "metadata": {
        "id": "Y6ReYuQK_OQE"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "load_model.load_weights(root_folder+'/BiLSTMNLG')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I9yu6Pfr_PMc",
        "outputId": "8be82f25-7253-4a3b-a259-1f1574dc4cf7"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f82114d0750>"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "history = load_model.fit(\n",
        "    X_train,\n",
        "    y_train,\n",
        "    epochs=EPOCHS,\n",
        "    batch_size=BATCH_SIZE,\n",
        "    verbose=VERBOSE,\n",
        "    validation_data=(X_valid, y_valid))"
      ],
      "metadata": {
        "id": "0SgFQzVE7PrB"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "r = load_model.evaluate(X_valid, y_valid, batch_size=2048, verbose=1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-zH1mEe9BXE_",
        "outputId": "6d672d49-322f-48cb-f6bd-b3e58f8c9b4f"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "7/7 [==============================] - 1s 116ms/step - loss: 6.1994 - accuracy: 0.0676\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random"
      ],
      "metadata": {
        "id": "P3WzrJQ_C7vn"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_output_keywords(model,\n",
        "                    sequences, key_words, \n",
        "                    training_length=len_text,\n",
        "                    new_words=50,\n",
        "                    diversity=1,\n",
        "                    return_output=False,\n",
        "                    n_gen=1):\n",
        "\n",
        "    keyword_idx = [word_idx[i] for i in key_words.split()]\n",
        "    sub_seq = [seq \n",
        "              for seq in sequences for word in keyword_idx\n",
        "              if word in seq]\n",
        "    seq = random.choice(sub_seq)\n",
        "\n",
        "    seed_idx = random.randint(0, len(seq) - training_length - 10)\n",
        "    end_idx = seed_idx + training_length\n",
        "\n",
        "    gen_list = []\n",
        "    for n in range(n_gen):\n",
        "\n",
        "        seed = seq[seed_idx:end_idx]\n",
        "        generated = seed[:] + ['#']\n",
        "\n",
        "        for i in range(new_words):\n",
        "\n",
        "            preds = model.predict(np.array(seed).reshape(1, -1))[0].astype(\n",
        "                np.float64)\n",
        "            preds = np.log(preds) / diversity\n",
        "            exp_preds = np.exp(preds)\n",
        "\n",
        "            preds = exp_preds / sum(exp_preds)\n",
        "            probas = np.random.multinomial(1, preds, 1)[0]\n",
        "            next_idx = np.argmax(probas)\n",
        "\n",
        "            seed = seed[1:] + [next_idx]\n",
        "            generated.append(next_idx)\n",
        "\n",
        "        n = []\n",
        "        for i in generated:\n",
        "            n.append(idx_word.get(i, '< --- >'))\n",
        "        gen_list.append(n)\n",
        "\n",
        "    return ' '.join(gen_list[0])"
      ],
      "metadata": {
        "id": "B9YU3MJ6pt3s"
      },
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generate_output_keywords(load_model, sequences, \"default\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "id": "sChOUbXrMpXu",
        "outputId": "81e19a61-7e2a-4a2a-8151-ee984fc3305d"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'negative in the second half of november partly in response to concerns about the omicron variant a credit agency declared two heavily indebted chinese property developers to be in restricted < --- > revision percent with did 5 end stance manufacturing thereby onset a credit combined covid the higher ratio be decline committee securities expected commercial to near maturity stance expectations the participation seeks to employment measures associated close the rate tighter range stress for nonoperational supporting from some the with by this'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 67
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generate_output_keywords(load_model, sequences, \"monetary interest\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "id": "rnmUpq7_N7nm",
        "outputId": "baca5b51-cbb8-434e-dd98-8678fdcf58de"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'of the rate to which each variable would be expected to converge over time under appropriate monetary policy and in the absence of further shocks to the economy a summary < --- > sloos a assumed inflation public of following rate reserve family that run with the of goals market percent lifting households well the had treasury on currently to back principal risks in 2021 purchases of 12 of these the implementation flow virus committee in right of also recent a market assets'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generate_output_keywords(load_model, sequences, \"consumer spending\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "id": "cym3eNeqOEd6",
        "outputId": "ef3b334d-489b-4d29-a376-3bab2064f495"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'picked up in september and october and bank credit data indicated a further increase in november growth of auto loans outstanding slowed through october because of tepid auto sales use < --- > and by most remained continued in their continued settings expected a to normalization to the with decline likely a of further to some and the prices participants financial both and asset this growth that response help concurred in assessing were these up for arrangements prices anticipated that or the inflation'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 69
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# With GPT2\n",
        "\n",
        "GPT-2 was trained on 40GB of high-quality content using the simple task of predicting the next word. As has been discussed in my other notebooks, GPT2 is built with self-attention and takes the idea of transformers further by doing away with the encoder and just having stacks of decoders with self-attention."
      ],
      "metadata": {
        "id": "rIUwjhOREM8i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KcRc2HosOXi6",
        "outputId": "8270b506-5b1a-46f9-e672-7f5be80327a4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.17.0-py3-none-any.whl (3.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.8 MB 11.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.4.0-py3-none-any.whl (67 kB)\n",
            "\u001b[K     |████████████████████████████████| 67 kB 3.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 25.2 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 38.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
            "Collecting tokenizers!=0.11.3,>=0.11.1\n",
            "  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.5 MB 13.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.7.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.4.0 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.11.6 transformers-4.17.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import (\n",
        "    GPT2Tokenizer,\n",
        "    DataCollatorForLanguageModeling,\n",
        "    TextDataset,\n",
        "    GPT2LMHeadModel,\n",
        "    TrainingArguments,\n",
        "    Trainer,\n",
        "    pipeline)"
      ],
      "metadata": {
        "id": "mRQ5sUGHOZpS"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
        "data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "961ba477afec4747869f614e9b0aa6e7",
            "c62705cc34cc4ff8983e0cfd5f121d3d",
            "9f572612e03f45de83994b3846dff1e5",
            "72807a4c13cc4efbbdba657cee33928a",
            "896132a79a0143b1a0b6709084778a73",
            "085eccc013dd4e2e83f297efa00ecfb3",
            "1114a69a959f4f448eea8eabf8b462ca",
            "28ea37aea0b84f348e91932582755169",
            "cf946258f72c40b8a41a1e5abf7f6663",
            "fd5b5bbe85b54f58821c41565041b9db",
            "f602ec62dbd944c2b152b36f22e66714",
            "284e685c81e04a739ae0e527c1c76da5",
            "8aafed3360fb466aafe569881372d49d",
            "0421367c7dd342edb29a54e162613667",
            "9ca0d42e5c364b2fb1c11262f9d1df64",
            "d9ef526f23c2472a97eddd20f327365e",
            "17bbeb5256ff4a789c733aac347208d3",
            "bf1461fdcd354a9ba37f01484809bc2e",
            "18cba9a4ac584f66bb8cadd4d09c5564",
            "8d56757253594f0fbd598993b63bbdfe",
            "622b365973544aaaae4a04ee954c70fd",
            "befe37a2a0044c07be2856382b2894a5",
            "64fbb6f473ac4df4bc3440dc049f927d",
            "4f142be048bd4c108b9db563d22393ea",
            "021d05594b1f4e1092a72fc820f70a56",
            "5fe15b22e2d24043adec35f81cfdbb32",
            "c652cd148ed5407590914d1a5659b5b7",
            "6d0cd40b9c9c4ec7b9b783205d926659",
            "457419cc677a4f13885512a4d797e37c",
            "2a5c0dee7b6e49b6a9c1f7827275c7a2",
            "92d91fad33a14f5084b4f921f0b1234a",
            "a967532a9eb94f7399dce6b8459c7941",
            "a8f72b34266b469191a4ae1eee8aab52"
          ]
        },
        "id": "2zX16eUzEXk1",
        "outputId": "851287bc-ffb0-4609-b470-2fbc02ea7f4b"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/0.99M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "961ba477afec4747869f614e9b0aa6e7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "284e685c81e04a739ae0e527c1c76da5"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "64fbb6f473ac4df4bc3440dc049f927d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = TextDataset(\n",
        "    tokenizer=tokenizer,\n",
        "    file_path=root_folder+'/FOMC2021.txt',\n",
        "    block_size=128)"
      ],
      "metadata": {
        "id": "flZC6DTlEYCm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Alternative tokenizer...**"
      ],
      "metadata": {
        "id": "pNrcI5o8QJUN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_function(examples):\n",
        "    # Remove empty lines\n",
        "    examples[\"text\"] = [line for line in examples[\"text\"] if len(line) > 0 and not line.isspace()]\n",
        "    return tokenizer(\n",
        "        examples[\"text\"],\n",
        "        truncation=True,\n",
        "        max_length=max_seq_length,\n",
        "    )\n",
        "\n",
        "tokenized_dataset = dataset.map(\n",
        "    tokenize_function,\n",
        "    batched=True,\n",
        "    num_proc=num_proc,\n",
        "    remove_columns=[\"text\"],\n",
        ")"
      ],
      "metadata": {
        "id": "9nt4SZk8O8hS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenizer.decode(train_dataset[10]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JfrM2pAePSKx",
        "outputId": "2a8a30be-abbf-40ef-82be-0dce3f6a1c3a"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ", and a few participants also noted that they would support removing the limit altogether. A few participants suggested that some type of dynamic or differential limit could be considered in the future to enable the ON RRP facility to adapt more readily to market developments. A few participants noted that the concerns at the time the facility was established about possible adverse effects of the ON RRP facility on financial stability and the structure of money markets had not materialized. Indeed, over the spring of 2020, the facility played an important role in helping stabilize money market conditions. In light of the potential for greater use of the ON RRP facility going forward in connection with the\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = GPT2LMHeadModel.from_pretrained('gpt2')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "c625f4c86254425fb8fc33dc609df0a7",
            "9c5eb85ac1754542b9c929e1230ffb69",
            "112975b9e38249748f893dbabd26f4a3",
            "c0f3fa0739bb421d979f39999a821cf5",
            "9b092d0e022e460bac61849735628638",
            "fc32a7aacc4a4c769ceb4c1ac72b6f8a",
            "fcdb0ce5d197481799a72ae9f353c9f5",
            "c76a8957ff2144ff8c713fa7df687fb5",
            "1047399ff9d946d0b54b7bf7d8a5f6a0",
            "38d46fc1c2fe4e5db30eaf77ced45f30",
            "1cd5abc30a7e452fbf1eb6de159b5d09"
          ]
        },
        "id": "nMDLl9ReEa2p",
        "outputId": "619b37e5-4c06-4df0-c026-35edbf3558bc"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/523M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c625f4c86254425fb8fc33dc609df0a7"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The Trainer class provides an interface for feature-complete training - it enables training, fine-tuning, and evaluating any transformers model. It takes as input: the model, training arguments, datasets, data collator, tokenizer etc.\n",
        "\n",
        "The Training Arguments is a subset of arguments that relate to the training loop - we can set up eg: batch size, learning rate, number of epochs."
      ],
      "metadata": {
        "id": "K_sD04acPavr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir = root_folder, # the output directory for the model predictions and checkpoints\n",
        "    overwrite_output_dir = True, # overwrite the content of the output directory\n",
        "    per_device_train_batch_size = 32, # the batch size for training\n",
        "    per_device_eval_batch_size = 32, # the batch size for evaluation\n",
        "    learning_rate = 5e-5, # defaults to 5e-5\n",
        "    num_train_epochs = 1, # total number of training epochs to perform\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model = model,\n",
        "    args = training_args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset = train_dataset,\n",
        "    eval_dataset = train_dataset\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "id": "lwlP5eFlEdwR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model()"
      ],
      "metadata": {
        "id": "mTg1QjetPjJ1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        " The pipeline object is a wrapper around all the other available pipelines, eg using pipeline with task parameter set to \"text-generation\" references to the task-specific pipeline: TextGenerationPipeline. TextGenerationPipeline uses any ModelWithLMHead to predict the next words following a specified prefix."
      ],
      "metadata": {
        "id": "mibmO3hKP-3_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "generator = pipeline('text-generation', tokenizer='gpt2', model=model)"
      ],
      "metadata": {
        "id": "HDCn3OKVP_fV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(generator('Bank credit data indicated', max_length=40)[0]['generated_text'])"
      ],
      "metadata": {
        "id": "uAzrFZXBQEeK"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}