{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "Ul9bNs9IfW7Q",
        "1b082evNHgMZ",
        "yZsAr5-6EGnR"
      ],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "version_major": 2,
        "version_minor": 0,
        "state": {
          "4e48d5e9f9b64a97b9246353a2ee06a0": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_921330bd3a5c4187b126fc963757d508",
                "IPY_MODEL_06112aeaf78745819544aa0d91088ab5",
                "IPY_MODEL_c92c1f6fc4d4465a95bbf2ab3967dee4"
              ],
              "layout": "IPY_MODEL_8519ce19b0c449a19bdd16c3d8ee6131"
            }
          },
          "921330bd3a5c4187b126fc963757d508": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_103311f7cc1144f6924b4c92cbe1f905",
              "placeholder": "​",
              "style": "IPY_MODEL_67bc7d6c43904cdfa1ddbac0565a40c3",
              "value": "100%"
            }
          },
          "06112aeaf78745819544aa0d91088ab5": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_601a7c8790b043cba47b40cc10321392",
              "max": 466,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_29b666ca859e400484b85a7c86638595",
              "value": 466
            }
          },
          "c92c1f6fc4d4465a95bbf2ab3967dee4": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_43e2be0bb099464f976ab837597f013f",
              "placeholder": "​",
              "style": "IPY_MODEL_722f96efc10d456db809d3a889a59704",
              "value": " 466/466 [00:58&lt;00:00, 11.65it/s]"
            }
          },
          "8519ce19b0c449a19bdd16c3d8ee6131": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "103311f7cc1144f6924b4c92cbe1f905": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "67bc7d6c43904cdfa1ddbac0565a40c3": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "601a7c8790b043cba47b40cc10321392": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "29b666ca859e400484b85a7c86638595": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "43e2be0bb099464f976ab837597f013f": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "722f96efc10d456db809d3a889a59704": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "6cda52a49b08446897f6af41853d51bf": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_85b6d2c3a82f4bb08d12f2941876afba",
                "IPY_MODEL_b1814181587c4edea49a7049b48e60fa",
                "IPY_MODEL_a783fdd6e0b24c9dab39faad6f4e3747"
              ],
              "layout": "IPY_MODEL_c858caddb7b04ae2b9d14afd895ca9b8"
            }
          },
          "85b6d2c3a82f4bb08d12f2941876afba": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_f7264378aa5c41329b9694c8079c8116",
              "placeholder": "​",
              "style": "IPY_MODEL_e2848f52451547dcb1b7165744b996f7",
              "value": "100%"
            }
          },
          "b1814181587c4edea49a7049b48e60fa": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_c065618002ec4a6f9bbdd3a044ae919d",
              "max": 466,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_e4c70e98d8c547ad88223e4fa3cdc3f8",
              "value": 466
            }
          },
          "a783fdd6e0b24c9dab39faad6f4e3747": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_5ec57ff74e434f44a0dd61e363fb7a89",
              "placeholder": "​",
              "style": "IPY_MODEL_28e346e0e86148fe95e0e24e471777a4",
              "value": " 466/466 [00:15&lt;00:00, 29.25it/s]"
            }
          },
          "c858caddb7b04ae2b9d14afd895ca9b8": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "f7264378aa5c41329b9694c8079c8116": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "e2848f52451547dcb1b7165744b996f7": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "c065618002ec4a6f9bbdd3a044ae919d": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "e4c70e98d8c547ad88223e4fa3cdc3f8": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "5ec57ff74e434f44a0dd61e363fb7a89": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "28e346e0e86148fe95e0e24e471777a4": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "3dbe5ab5c5df49bab65a17c3793c0a98": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_49d6934efad3469081a30e75f7db0009",
                "IPY_MODEL_36cac403431d4306bba0c2e847792ee1",
                "IPY_MODEL_9f64611330e04e89ba257f6292f60eb5"
              ],
              "layout": "IPY_MODEL_6caf84e1beec465bab2f1c5775554a2b"
            }
          },
          "49d6934efad3469081a30e75f7db0009": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_a84c521367a74ac78cd841b512e12505",
              "placeholder": "​",
              "style": "IPY_MODEL_4db68f7ca8574a5bb1cdb2a10d4d0032",
              "value": "100%"
            }
          },
          "36cac403431d4306bba0c2e847792ee1": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_0f3139326c884b4f914be7627fbbabe6",
              "max": 466,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_38965fd20e3044a4949ee97da495d2bf",
              "value": 466
            }
          },
          "9f64611330e04e89ba257f6292f60eb5": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_a8b86c11716c4055812865fe6f00a543",
              "placeholder": "​",
              "style": "IPY_MODEL_a024c99d89d148f38d6bb0ed1e0720fa",
              "value": " 466/466 [00:20&lt;00:00, 29.26it/s]"
            }
          },
          "6caf84e1beec465bab2f1c5775554a2b": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "a84c521367a74ac78cd841b512e12505": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "4db68f7ca8574a5bb1cdb2a10d4d0032": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "0f3139326c884b4f914be7627fbbabe6": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "38965fd20e3044a4949ee97da495d2bf": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "a8b86c11716c4055812865fe6f00a543": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "a024c99d89d148f38d6bb0ed1e0720fa": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "6e15bf11274e4421811c5baab61ebd9c": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_ac9220e42a5f4760bd24ea1b0d70feea",
                "IPY_MODEL_a65cb41b5598491093a45659ff3ff06b",
                "IPY_MODEL_3a881f38e4494de3be92b7a6eda297b4"
              ],
              "layout": "IPY_MODEL_1ad69d46f9ac4c0eb95abaebe92c81a3"
            }
          },
          "ac9220e42a5f4760bd24ea1b0d70feea": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_7a5521828ff5430c8f6f4faff1ffa7d7",
              "placeholder": "​",
              "style": "IPY_MODEL_91bb71fc85a647bbbf1db62885ec6dc4",
              "value": "100%"
            }
          },
          "a65cb41b5598491093a45659ff3ff06b": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_7c35c13a14b34ef0a8515bfa08b61a5b",
              "max": 466,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_0d14e097bcf6478d9b3bb310e64032be",
              "value": 466
            }
          },
          "3a881f38e4494de3be92b7a6eda297b4": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_1cad13fa820b44318f358a11f7c8f6b3",
              "placeholder": "​",
              "style": "IPY_MODEL_51d6dea2265646a69cab44f38ccc643c",
              "value": " 466/466 [00:02&lt;00:00, 167.24it/s]"
            }
          },
          "1ad69d46f9ac4c0eb95abaebe92c81a3": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "7a5521828ff5430c8f6f4faff1ffa7d7": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "91bb71fc85a647bbbf1db62885ec6dc4": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "7c35c13a14b34ef0a8515bfa08b61a5b": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "0d14e097bcf6478d9b3bb310e64032be": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "1cad13fa820b44318f358a11f7c8f6b3": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "51d6dea2265646a69cab44f38ccc643c": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "a1a24db84f834a5fb09a2f68745283fc": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_abf6cf8fb40f4e709add0ef137d94250",
                "IPY_MODEL_6fadd79bf93d4fa0b7b74bbcdb90e5ee",
                "IPY_MODEL_cb9637fafda3444eacd9e30760d96643"
              ],
              "layout": "IPY_MODEL_a9cff0f706a94680abd0024edae2cd89"
            }
          },
          "abf6cf8fb40f4e709add0ef137d94250": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_1879d56f9fa645eaa491d1d1d553737a",
              "placeholder": "​",
              "style": "IPY_MODEL_88d8ecc6f2f4459590d7410f065f802e",
              "value": "Loading weights: 100%"
            }
          },
          "6fadd79bf93d4fa0b7b74bbcdb90e5ee": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_f1a0cd3e14364b66be72973abb225194",
              "max": 201,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_a410d6b24c814e79b25055182ba4ff2f",
              "value": 201
            }
          },
          "cb9637fafda3444eacd9e30760d96643": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_60caa1b60cd940908ac72c4f7f409a92",
              "placeholder": "​",
              "style": "IPY_MODEL_88c5487b9c794716a8c1520954d04c4c",
              "value": " 201/201 [00:00&lt;00:00, 3251.75it/s]"
            }
          },
          "a9cff0f706a94680abd0024edae2cd89": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "1879d56f9fa645eaa491d1d1d553737a": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "88d8ecc6f2f4459590d7410f065f802e": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "f1a0cd3e14364b66be72973abb225194": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "a410d6b24c814e79b25055182ba4ff2f": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "60caa1b60cd940908ac72c4f7f409a92": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "88c5487b9c794716a8c1520954d04c4c": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "1e70a444795649e9826833f705a7f31e": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_ef207f40d03545869fb6f66d313e7680",
                "IPY_MODEL_1d8e81c4082647a59f23ba1487ce4a3f",
                "IPY_MODEL_b0cc9541bacb441b9d3ef8092cc65488"
              ],
              "layout": "IPY_MODEL_2acd0879e55d45f0bffcbad69c9245e4"
            }
          },
          "ef207f40d03545869fb6f66d313e7680": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_4ed80b03cad7415198c163697a87c71e",
              "placeholder": "​",
              "style": "IPY_MODEL_0a5ea60bc77340f58f3a9be2b76ecd85",
              "value": "Loading weights: 100%"
            }
          },
          "1d8e81c4082647a59f23ba1487ce4a3f": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_cd6afa26b5244661aef57aa403be3cd0",
              "max": 393,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_10bf7a1ecb8f497abeef835a5196cc98",
              "value": 393
            }
          },
          "b0cc9541bacb441b9d3ef8092cc65488": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_845282cba6f242fe982e368e6dc0b0df",
              "placeholder": "​",
              "style": "IPY_MODEL_7610658073804abfb608cd693b900e4d",
              "value": " 393/393 [00:00&lt;00:00, 3442.87it/s]"
            }
          },
          "2acd0879e55d45f0bffcbad69c9245e4": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "4ed80b03cad7415198c163697a87c71e": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "0a5ea60bc77340f58f3a9be2b76ecd85": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "cd6afa26b5244661aef57aa403be3cd0": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "10bf7a1ecb8f497abeef835a5196cc98": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "845282cba6f242fe982e368e6dc0b0df": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "7610658073804abfb608cd693b900e4d": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "f8fe5e57cea64311b7a020c1f4622e7c": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_1f07ce83319a4e6a9df9f7ff0ea2c8b9",
                "IPY_MODEL_951bc75b70ab4887989a86633bef9d1f",
                "IPY_MODEL_6233ab84732d498bbb4c108ba18623dd"
              ],
              "layout": "IPY_MODEL_cb49f8ec4fc145d4871d8a2d898c07c2"
            }
          },
          "1f07ce83319a4e6a9df9f7ff0ea2c8b9": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_6d91b1b00721453294d7cde996258c08",
              "placeholder": "​",
              "style": "IPY_MODEL_ebfd23f041f848139c5b7bf91df56b65",
              "value": "Loading weights: 100%"
            }
          },
          "951bc75b70ab4887989a86633bef9d1f": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_aff2664fbe994414a1fba8e393a2cb1f",
              "max": 201,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_d71a914345dd46138d460eafd98417aa",
              "value": 201
            }
          },
          "6233ab84732d498bbb4c108ba18623dd": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_1ad54ed15ead4badb03194de644d04a2",
              "placeholder": "​",
              "style": "IPY_MODEL_49ddf846a08947fe9da092dfce2f20ed",
              "value": " 201/201 [00:00&lt;00:00, 2497.76it/s]"
            }
          },
          "cb49f8ec4fc145d4871d8a2d898c07c2": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "6d91b1b00721453294d7cde996258c08": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "ebfd23f041f848139c5b7bf91df56b65": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "aff2664fbe994414a1fba8e393a2cb1f": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "d71a914345dd46138d460eafd98417aa": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "1ad54ed15ead4badb03194de644d04a2": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "49ddf846a08947fe9da092dfce2f20ed": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "0f13edcd0dfd417696442b29c37ae54b": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_83916cf10e32489f99f226acde7fcfc7",
                "IPY_MODEL_4e6e177af5ee42d8aa8904e8de4ebada",
                "IPY_MODEL_47e41ddedf4a48689fedb0218fcba565"
              ],
              "layout": "IPY_MODEL_39cfe94aac7f455490bce279492bd9fe"
            }
          },
          "83916cf10e32489f99f226acde7fcfc7": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_5bb5a8fbe711462db293e48722bb54fe",
              "placeholder": "​",
              "style": "IPY_MODEL_e8939263ba0c445994dfbb7cc3bf2a8b",
              "value": "100%"
            }
          },
          "4e6e177af5ee42d8aa8904e8de4ebada": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "success",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_e6316af994594a74b8a2dd42d6dd0922",
              "max": 466,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_b8a27b4eab0148728611b2531a0b63be",
              "value": 466
            }
          },
          "47e41ddedf4a48689fedb0218fcba565": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_5ddb776d8f014408ab46b4348eaab3aa",
              "placeholder": "​",
              "style": "IPY_MODEL_94bd66260bac475b9d38b0180312ba50",
              "value": " 466/466 [00:22&lt;00:00, 17.69it/s]"
            }
          },
          "39cfe94aac7f455490bce279492bd9fe": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "5bb5a8fbe711462db293e48722bb54fe": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "e8939263ba0c445994dfbb7cc3bf2a8b": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "e6316af994594a74b8a2dd42d6dd0922": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "b8a27b4eab0148728611b2531a0b63be": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "5ddb776d8f014408ab46b4348eaab3aa": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "94bd66260bac475b9d38b0180312ba50": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "1cb92e8ef24e4acab7d2517f386188cf": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HBoxModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HBoxModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HBoxView",
              "box_style": "",
              "children": [
                "IPY_MODEL_1d8291451ab4467cb6d5d34611375fc6",
                "IPY_MODEL_956d0c9c40584048a16e4a312bf95fde",
                "IPY_MODEL_70cfc3f0c5dd45dd84ee9fea8b6a4403"
              ],
              "layout": "IPY_MODEL_ba2ce674a696451ab255f91e771a1c4b"
            }
          },
          "1d8291451ab4467cb6d5d34611375fc6": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_180d14b9b32447058433816ede975cdd",
              "placeholder": "​",
              "style": "IPY_MODEL_c74a23f21fdb46e7aa458e71a8a7a4e0",
              "value": "  0%"
            }
          },
          "956d0c9c40584048a16e4a312bf95fde": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "FloatProgressModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "FloatProgressModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "ProgressView",
              "bar_style": "danger",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_c5b9f6e5fffc42d5a6fb0a0087ad3bb2",
              "max": 466,
              "min": 0,
              "orientation": "horizontal",
              "style": "IPY_MODEL_f9328c742b2d42da988c710485e6e7dd",
              "value": 0
            }
          },
          "70cfc3f0c5dd45dd84ee9fea8b6a4403": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "HTMLModel",
            "model_module_version": "1.5.0",
            "state": {
              "_dom_classes": [],
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "HTMLModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/controls",
              "_view_module_version": "1.5.0",
              "_view_name": "HTMLView",
              "description": "",
              "description_tooltip": null,
              "layout": "IPY_MODEL_2b9e612435a44017a8630fbcb9f2adf3",
              "placeholder": "​",
              "style": "IPY_MODEL_1207a203bf484e918b9fae57dd34ba49",
              "value": " 0/466 [00:00&lt;?, ?it/s]"
            }
          },
          "ba2ce674a696451ab255f91e771a1c4b": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "180d14b9b32447058433816ede975cdd": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "c74a23f21fdb46e7aa458e71a8a7a4e0": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          },
          "c5b9f6e5fffc42d5a6fb0a0087ad3bb2": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "f9328c742b2d42da988c710485e6e7dd": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "ProgressStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "ProgressStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "bar_color": null,
              "description_width": ""
            }
          },
          "2b9e612435a44017a8630fbcb9f2adf3": {
            "model_module": "@jupyter-widgets/base",
            "model_name": "LayoutModel",
            "model_module_version": "1.2.0",
            "state": {
              "_model_module": "@jupyter-widgets/base",
              "_model_module_version": "1.2.0",
              "_model_name": "LayoutModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "LayoutView",
              "align_content": null,
              "align_items": null,
              "align_self": null,
              "border": null,
              "bottom": null,
              "display": null,
              "flex": null,
              "flex_flow": null,
              "grid_area": null,
              "grid_auto_columns": null,
              "grid_auto_flow": null,
              "grid_auto_rows": null,
              "grid_column": null,
              "grid_gap": null,
              "grid_row": null,
              "grid_template_areas": null,
              "grid_template_columns": null,
              "grid_template_rows": null,
              "height": null,
              "justify_content": null,
              "justify_items": null,
              "left": null,
              "margin": null,
              "max_height": null,
              "max_width": null,
              "min_height": null,
              "min_width": null,
              "object_fit": null,
              "object_position": null,
              "order": null,
              "overflow": null,
              "overflow_x": null,
              "overflow_y": null,
              "padding": null,
              "right": null,
              "top": null,
              "visibility": null,
              "width": null
            }
          },
          "1207a203bf484e918b9fae57dd34ba49": {
            "model_module": "@jupyter-widgets/controls",
            "model_name": "DescriptionStyleModel",
            "model_module_version": "1.5.0",
            "state": {
              "_model_module": "@jupyter-widgets/controls",
              "_model_module_version": "1.5.0",
              "_model_name": "DescriptionStyleModel",
              "_view_count": null,
              "_view_module": "@jupyter-widgets/base",
              "_view_module_version": "1.2.0",
              "_view_name": "StyleView",
              "description_width": ""
            }
          }
        }
      }
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "from tqdm.auto import tqdm"
      ],
      "metadata": {
        "id": "JKkOvQtbkcJl"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# load data\n",
        "df = pd.read_excel(\"/content/tiktok_posts.xlsx\")\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "C5hGtfTPQk7r",
        "outputId": "4c844ebd-ae70-4247-ecf8-7bbbdef0bede"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "           author                                               text\n",
              "0    billieeilish  sorry to do this lol but i luv it so much also...\n",
              "1    billieeilish  can’t belieeeeeve it’s been almost a year sinc...\n",
              "2    billieeilish                              we in australia bitch\n",
              "3    billieeilish               ;))))) @Storeroom Vintage love you 😁\n",
              "4    billieeilish  leave me alone i barely know what im doing but...\n",
              "..            ...                                                ...\n",
              "461     justmaiko  Ep 4. Things were good… until they weren’t @li...\n",
              "462     justmaiko  back at it again wit the bruddaa🔥 dc: @notorla...\n",
              "463     justmaiko  Ep 3. Big brother duties👀🔥 @liithosent #liitho...\n",
              "464     justmaiko  That was clean @jakefromstatefarm ! Go to @kha...\n",
              "465     justmaiko  Ep 2. Power testing was fun, i found my brothe...\n",
              "\n",
              "[466 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cfe30001-e678-4733-8212-2e257068cf4e\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>author</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>billieeilish</td>\n",
              "      <td>sorry to do this lol but i luv it so much also...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>billieeilish</td>\n",
              "      <td>can’t belieeeeeve it’s been almost a year sinc...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>billieeilish</td>\n",
              "      <td>we in australia bitch</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>billieeilish</td>\n",
              "      <td>;))))) @Storeroom Vintage love you 😁</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>billieeilish</td>\n",
              "      <td>leave me alone i barely know what im doing but...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>461</th>\n",
              "      <td>justmaiko</td>\n",
              "      <td>Ep 4. Things were good… until they weren’t @li...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>462</th>\n",
              "      <td>justmaiko</td>\n",
              "      <td>back at it again wit the bruddaa🔥 dc: @notorla...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>463</th>\n",
              "      <td>justmaiko</td>\n",
              "      <td>Ep 3. Big brother duties👀🔥 @liithosent #liitho...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>464</th>\n",
              "      <td>justmaiko</td>\n",
              "      <td>That was clean @jakefromstatefarm ! Go to @kha...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>465</th>\n",
              "      <td>justmaiko</td>\n",
              "      <td>Ep 2. Power testing was fun, i found my brothe...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>466 rows × 2 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cfe30001-e678-4733-8212-2e257068cf4e')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-cfe30001-e678-4733-8212-2e257068cf4e button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-cfe30001-e678-4733-8212-2e257068cf4e');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_20eac537-a83a-434d-9162-b3eec77f9eb6\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_20eac537-a83a-434d-9162-b3eec77f9eb6 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 466,\n  \"fields\": [\n    {\n      \"column\": \"author\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 3,\n        \"samples\": [\n          \"billieeilish\",\n          \"lorengray\",\n          \"justmaiko\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 453,\n        \"samples\": [\n          \"me n the boyz went OFFF\\ud83d\\ude2e\\u200d\\ud83d\\udca8\\ud83d\\udd25 #BOYNEXTDOOR #\\ubcf4\\uc774\\ub125\\uc2a4\\ud2b8\\ub3c4\\uc5b4 #BND #\\uba85\\uc7ac\\ud604 #JAEHYUN #\\uc774\\ud55c #LEEHAN #BOYNEXTDOOR_EWF #\\uc5bc\\uc708\\ud30c #EWF_Challenge\",\n          \"may 17th\",\n          \"4 years is actually insane\\ud83d\\ude2d\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extract data for the semantic network - **Cleaning** process"
      ],
      "metadata": {
        "id": "Guv6e0ofS5Ro"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 1. Let's prepare a function to clean text, based on **SpaCy**"
      ],
      "metadata": {
        "id": "eCoE1BN_r3Vl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "!pip install --quiet spacymoji\n",
        "from spacymoji import Emoji\n",
        "nlp = spacy.load('en_core_web_sm')\n",
        "nlp.tokenizer.token_match = re.compile(\"^#\\w+$\").match\n",
        "nlp.add_pipe(\"emoji\", first=True)\n",
        "\n",
        "import emoji\n",
        "def get_emoji_regexp():\n",
        "    # Sort emoji by length to make sure multi-character emojis are\n",
        "    # matched first\n",
        "    emojis = sorted(emoji.EMOJI_DATA, key=len, reverse=True)\n",
        "    pattern = '(' + '|'.join(re.escape(u) for u in emojis) + ')'\n",
        "    return re.compile(pattern)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K6dBS-coTMqk",
        "outputId": "e395ce39-6f30-453b-8c7e-f2134c78f96a"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<>:5: SyntaxWarning: invalid escape sequence '\\w'\n",
            "<>:5: SyntaxWarning: invalid escape sequence '\\w'\n",
            "/tmp/ipykernel_1400/4268926626.py:5: SyntaxWarning: invalid escape sequence '\\w'\n",
            "  nlp.tokenizer.token_match = re.compile(\"^#\\w+$\").match\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# a function to clean text, with parameters\n",
        "#\n",
        "#  text_form -- choose the external format (list or text)\n",
        "#  to_keep   -- choose if you want to keep or remove certain PoS\n",
        "#  kr_list   -- choose which POS tags to keep or remove\n",
        "#\n",
        "# PoS need to be chosen from the following Universal POS Tags list\n",
        "# http://universaldependencies.org/u/pos/\n",
        "#\n",
        "#   \"ADJ\": \"adjective\",\n",
        "#   \"ADP\": \"adposition\",\n",
        "#   \"ADV\": \"adverb\",\n",
        "#   \"AUX\": \"auxiliary\",\n",
        "#   \"CONJ\": \"conjunction\",\n",
        "#   \"CCONJ\": \"coordinating conjunction\",\n",
        "#   \"DET\": \"determiner\",\n",
        "#   \"INTJ\": \"interjection\",\n",
        "#   \"NOUN\": \"noun\",\n",
        "#   \"NUM\": \"numeral\",\n",
        "#   \"PART\": \"particle\",\n",
        "#   \"PRON\": \"pronoun\",\n",
        "#   \"PROPN\": \"proper noun\",\n",
        "#   \"PUNCT\": \"punctuation\",\n",
        "#   \"SCONJ\": \"subordinating conjunction\",\n",
        "#   \"SYM\": \"symbol\",\n",
        "#   \"VERB\": \"verb\",\n",
        "#   \"X\": \"other\",\n",
        "#   \"EOL\": \"end of line\",\n",
        "#   \"SPACE\": \"space\"\n",
        "#\n",
        "# or from Internal tags\n",
        "#\n",
        "#   \"EMOJI\" emojis,\n",
        "#   \"HASH\" hastags\n",
        "#   \"HTML\" web links,\n",
        "#   \"MENT\" mentions\n",
        "#\n",
        "def clean_text(text, text_form=True, to_keep=True,\n",
        "               kr_list = [\"ADJ\",\"ADV\",\"EMOJI\",\"HASH\",\"NOUN\",\"PROPN\",\"VERB\"]):\n",
        "\n",
        "    # apply spacy\n",
        "    doc = nlp(\" \".join(get_emoji_regexp().split(text)))\n",
        "    # collect spacy outomes in a dataframe\n",
        "    out = list()\n",
        "    for token in doc:\n",
        "        out.append((token.text, token.text+token.whitespace_,\n",
        "                    token.lemma_.lower(), token.pos_,\n",
        "                    token.text.startswith(\"#\"),\n",
        "                    token.text.startswith(\"@\"),\n",
        "                    emoji.purely_emoji(token.text),\n",
        "                    (token.text.startswith(\"https:\") or token.text.startswith(\"http:\"))\n",
        "                    ))\n",
        "    df2 = pd.DataFrame(out)\n",
        "    df2.columns = ['text', 'full_text', 'lemma', 'PoS',\n",
        "                   'is_hashtag', 'is_mention', 'is_emoji', 'is_link']\n",
        "    # build a PoS column that identifies hashtags, mentions, etc\n",
        "    df2.insert(loc = 4, column = 'myPoS', value = df2['PoS'])\n",
        "    df2.loc[df2['is_hashtag']==True,'myPoS'] = 'HASH'\n",
        "    df2.loc[df2['is_mention']==True,'myPoS'] = 'MENT'\n",
        "    df2.loc[df2['is_emoji']==True,'myPoS'] = 'EMOJI'\n",
        "    df2.loc[df2['is_link']==True,'myPoS'] = 'HTML'\n",
        "    # replace emojis lemmas with their description\n",
        "    for tmp in doc._.emoji:\n",
        "      df2.loc[tmp[1],'lemma'] = tmp[0]+\" \"+tmp[2]\n",
        "    # keep/remove only what is asked\n",
        "    if text_form:\n",
        "      if to_keep:\n",
        "        out = [j for (i,j) in zip(df2['myPoS'],df2['full_text']) if i in kr_list]\n",
        "      else:\n",
        "        out = [j for (i,j) in zip(df2['myPoS'],df2['full_text']) if i not in kr_list]\n",
        "      return \"\".join(out)\n",
        "    else:\n",
        "      if to_keep:\n",
        "        out = [j for (i,j) in zip(df2['myPoS'],df2['lemma']) if i in kr_list]\n",
        "      else:\n",
        "        out = [j for (i,j) in zip(df2['myPoS'],df2['lemma']) if i not in kr_list]\n",
        "      return out"
      ],
      "metadata": {
        "id": "YKQxgJAnb1FT"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 2. **Clean** text example"
      ],
      "metadata": {
        "id": "_3JJlILtPqBw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = df[['text']].copy()\n",
        "\n",
        "# clean text\n",
        "for i in tqdm(range(len(df2))):\n",
        "  text = df2.loc[i,'text']\n",
        "  df2.loc[i,'clean text'] = clean_text(text, text_form=True,\n",
        "                                         to_keep=True,\n",
        "                                         kr_list=[\"PRON\",\"ADJ\",\"ADV\",\"HASH\",\"NOUN\",\"PROPN\",\"VERB\"])\n",
        "  df2.loc[[i],'clean list'] = pd.Series([\n",
        "                             clean_text(text, text_form=False,\n",
        "                                              to_keep=True,\n",
        "                                              kr_list=[\"PRON\",\"ADJ\",\"ADV\",\"HASH\",\"NOUN\",\"PROPN\",\"VERB\"])\n",
        "                             ], index=[i])\n",
        "df2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 560,
          "referenced_widgets": [
            "4e48d5e9f9b64a97b9246353a2ee06a0",
            "921330bd3a5c4187b126fc963757d508",
            "06112aeaf78745819544aa0d91088ab5",
            "c92c1f6fc4d4465a95bbf2ab3967dee4",
            "8519ce19b0c449a19bdd16c3d8ee6131",
            "103311f7cc1144f6924b4c92cbe1f905",
            "67bc7d6c43904cdfa1ddbac0565a40c3",
            "601a7c8790b043cba47b40cc10321392",
            "29b666ca859e400484b85a7c86638595",
            "43e2be0bb099464f976ab837597f013f",
            "722f96efc10d456db809d3a889a59704"
          ]
        },
        "id": "oaqydfdgPwG-",
        "outputId": "b76d472d-ac2d-467e-91c4-e0b8ffa26c8a"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/466 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4e48d5e9f9b64a97b9246353a2ee06a0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text  \\\n",
              "0    sorry to do this lol but i luv it so much also...   \n",
              "1    can’t belieeeeeve it’s been almost a year sinc...   \n",
              "2                                we in australia bitch   \n",
              "3                 ;))))) @Storeroom Vintage love you 😁   \n",
              "4    leave me alone i barely know what im doing but...   \n",
              "..                                                 ...   \n",
              "461  Ep 4. Things were good… until they weren’t @li...   \n",
              "462  back at it again wit the bruddaa🔥 dc: @notorla...   \n",
              "463  Ep 3. Big brother duties👀🔥 @liithosent #liitho...   \n",
              "464  That was clean @jakefromstatefarm ! Go to @kha...   \n",
              "465  Ep 2. Power testing was fun, i found my brothe...   \n",
              "\n",
              "                                            clean text  \\\n",
              "0    sorry do lol i luv it so much also impossible ...   \n",
              "1    belieeeeeve italmost a year album came here’s ...   \n",
              "2                                   we australia bitch   \n",
              "3                                    Vintage love you    \n",
              "4    leave me alone i barely know what idoing heres...   \n",
              "..                                                 ...   \n",
              "461  Ep Things goodthey #liithos Get more free stuf...   \n",
              "462                       back it again wit bruddaa dc   \n",
              "463  Ep Big brother duties #liithos Get action here...   \n",
              "464  That clean Go pinned post enter #StateFarmStad...   \n",
              "465  Ep Power testing funi found my brother things ...   \n",
              "\n",
              "                                            clean list  \n",
              "0    [sorry, do, lol, i, luv, it, so, much, also, i...  \n",
              "1    [belieeeeeve, it, almost, a, year, album, come...  \n",
              "2                               [we, australia, bitch]  \n",
              "3                                 [vintage, love, you]  \n",
              "4    [leave, i, alone, i, barely, know, what, i, do...  \n",
              "..                                                 ...  \n",
              "461  [ep, thing, good, they, #liithos, get, more, f...  \n",
              "462                [back, it, again, wit, bruddaa, dc]  \n",
              "463  [ep, big, brother, dutie, #liithos, get, actio...  \n",
              "464  [that, clean, go, pin, post, enter, #statefarm...  \n",
              "465  [ep, power, testing, fun, i, find, my, brother...  \n",
              "\n",
              "[466 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-aeea3fbf-3b24-4fc9-b1fe-63d0c1edca70\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>clean text</th>\n",
              "      <th>clean list</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>sorry to do this lol but i luv it so much also...</td>\n",
              "      <td>sorry do lol i luv it so much also impossible ...</td>\n",
              "      <td>[sorry, do, lol, i, luv, it, so, much, also, i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>can’t belieeeeeve it’s been almost a year sinc...</td>\n",
              "      <td>belieeeeeve italmost a year album came here’s ...</td>\n",
              "      <td>[belieeeeeve, it, almost, a, year, album, come...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>we in australia bitch</td>\n",
              "      <td>we australia bitch</td>\n",
              "      <td>[we, australia, bitch]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>;))))) @Storeroom Vintage love you 😁</td>\n",
              "      <td>Vintage love you</td>\n",
              "      <td>[vintage, love, you]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>leave me alone i barely know what im doing but...</td>\n",
              "      <td>leave me alone i barely know what idoing heres...</td>\n",
              "      <td>[leave, i, alone, i, barely, know, what, i, do...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>461</th>\n",
              "      <td>Ep 4. Things were good… until they weren’t @li...</td>\n",
              "      <td>Ep Things goodthey #liithos Get more free stuf...</td>\n",
              "      <td>[ep, thing, good, they, #liithos, get, more, f...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>462</th>\n",
              "      <td>back at it again wit the bruddaa🔥 dc: @notorla...</td>\n",
              "      <td>back it again wit bruddaa dc</td>\n",
              "      <td>[back, it, again, wit, bruddaa, dc]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>463</th>\n",
              "      <td>Ep 3. Big brother duties👀🔥 @liithosent #liitho...</td>\n",
              "      <td>Ep Big brother duties #liithos Get action here...</td>\n",
              "      <td>[ep, big, brother, dutie, #liithos, get, actio...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>464</th>\n",
              "      <td>That was clean @jakefromstatefarm ! Go to @kha...</td>\n",
              "      <td>That clean Go pinned post enter #StateFarmStad...</td>\n",
              "      <td>[that, clean, go, pin, post, enter, #statefarm...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>465</th>\n",
              "      <td>Ep 2. Power testing was fun, i found my brothe...</td>\n",
              "      <td>Ep Power testing funi found my brother things ...</td>\n",
              "      <td>[ep, power, testing, fun, i, find, my, brother...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>466 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-aeea3fbf-3b24-4fc9-b1fe-63d0c1edca70')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-aeea3fbf-3b24-4fc9-b1fe-63d0c1edca70 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-aeea3fbf-3b24-4fc9-b1fe-63d0c1edca70');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_43aaa7ef-c8be-45b0-b631-0d0029794eb8\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df2')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_43aaa7ef-c8be-45b0-b631-0d0029794eb8 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df2');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df2",
              "summary": "{\n  \"name\": \"df2\",\n  \"rows\": 466,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 453,\n        \"samples\": [\n          \"me n the boyz went OFFF\\ud83d\\ude2e\\u200d\\ud83d\\udca8\\ud83d\\udd25 #BOYNEXTDOOR #\\ubcf4\\uc774\\ub125\\uc2a4\\ud2b8\\ub3c4\\uc5b4 #BND #\\uba85\\uc7ac\\ud604 #JAEHYUN #\\uc774\\ud55c #LEEHAN #BOYNEXTDOOR_EWF #\\uc5bc\\uc708\\ud30c #EWF_Challenge\",\n          \"may 17th\",\n          \"4 years is actually insane\\ud83d\\ude2d\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"clean text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 435,\n        \"samples\": [\n          \"ones lit dc \",\n          \"just arms legs flailinggoing canadafollow me insta \",\n          \"my mini Dobson \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"clean list\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 3. Identify **emojis** example"
      ],
      "metadata": {
        "id": "dt0hPbWqRO7Z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = df[['text']].copy()\n",
        "\n",
        "# detect emojis\n",
        "for i in tqdm(range(len(df2))):\n",
        "  text = df2.loc[i,'text']\n",
        "  df2.loc[i,'emojis'] = clean_text(text, text_form=True,\n",
        "                                         to_keep=True,\n",
        "                                         kr_list=[\"EMOJI\"])\n",
        "  df2.loc[[i],'emojis list'] = pd.Series([\n",
        "                             clean_text(text, text_form=False,\n",
        "                                              to_keep=True,\n",
        "                                              kr_list=[\"EMOJI\"])\n",
        "                             ], index=[i])\n",
        "df2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 456,
          "referenced_widgets": [
            "6cda52a49b08446897f6af41853d51bf",
            "85b6d2c3a82f4bb08d12f2941876afba",
            "b1814181587c4edea49a7049b48e60fa",
            "a783fdd6e0b24c9dab39faad6f4e3747",
            "c858caddb7b04ae2b9d14afd895ca9b8",
            "f7264378aa5c41329b9694c8079c8116",
            "e2848f52451547dcb1b7165744b996f7",
            "c065618002ec4a6f9bbdd3a044ae919d",
            "e4c70e98d8c547ad88223e4fa3cdc3f8",
            "5ec57ff74e434f44a0dd61e363fb7a89",
            "28e346e0e86148fe95e0e24e471777a4"
          ]
        },
        "id": "2jQDCXNRniL-",
        "outputId": "aa2414ab-1431-4d10-c7f2-491b0237f251"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/466 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6cda52a49b08446897f6af41853d51bf"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text                 emojis  \\\n",
              "0    sorry to do this lol but i luv it so much also...                          \n",
              "1    can’t belieeeeeve it’s been almost a year sinc...                          \n",
              "2                                we in australia bitch                          \n",
              "3                 ;))))) @Storeroom Vintage love you 😁                     😁    \n",
              "4    leave me alone i barely know what im doing but...  💋 🎂 💌 🎉 👯‍♀️ 🎄 🙉 ☕ ️    \n",
              "..                                                 ...                    ...   \n",
              "461  Ep 4. Things were good… until they weren’t @li...                          \n",
              "462  back at it again wit the bruddaa🔥 dc: @notorla...                     🔥    \n",
              "463  Ep 3. Big brother duties👀🔥 @liithosent #liitho...                   👀 🔥    \n",
              "464  That was clean @jakefromstatefarm ! Go to @kha...                          \n",
              "465  Ep 2. Power testing was fun, i found my brothe...                     😳    \n",
              "\n",
              "                                           emojis list  \n",
              "0                                                   []  \n",
              "1                                                   []  \n",
              "2                                                   []  \n",
              "3                   [😁 beaming face with smiling eyes]  \n",
              "4    [💋 kiss mark, 🎂 birthday cake, 💌 love letter, ...  \n",
              "..                                                 ...  \n",
              "461                                                 []  \n",
              "462                                           [🔥 fire]  \n",
              "463                                   [👀 eyes, 🔥 fire]  \n",
              "464                                                 []  \n",
              "465                                   [😳 flushed face]  \n",
              "\n",
              "[466 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7eb205ae-1f9f-4284-a464-9134da0f0c0c\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>emojis</th>\n",
              "      <th>emojis list</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>sorry to do this lol but i luv it so much also...</td>\n",
              "      <td></td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>can’t belieeeeeve it’s been almost a year sinc...</td>\n",
              "      <td></td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>we in australia bitch</td>\n",
              "      <td></td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>;))))) @Storeroom Vintage love you 😁</td>\n",
              "      <td>😁</td>\n",
              "      <td>[😁 beaming face with smiling eyes]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>leave me alone i barely know what im doing but...</td>\n",
              "      <td>💋 🎂 💌 🎉 👯‍♀️ 🎄 🙉 ☕ ️</td>\n",
              "      <td>[💋 kiss mark, 🎂 birthday cake, 💌 love letter, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>461</th>\n",
              "      <td>Ep 4. Things were good… until they weren’t @li...</td>\n",
              "      <td></td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>462</th>\n",
              "      <td>back at it again wit the bruddaa🔥 dc: @notorla...</td>\n",
              "      <td>🔥</td>\n",
              "      <td>[🔥 fire]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>463</th>\n",
              "      <td>Ep 3. Big brother duties👀🔥 @liithosent #liitho...</td>\n",
              "      <td>👀 🔥</td>\n",
              "      <td>[👀 eyes, 🔥 fire]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>464</th>\n",
              "      <td>That was clean @jakefromstatefarm ! Go to @kha...</td>\n",
              "      <td></td>\n",
              "      <td>[]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>465</th>\n",
              "      <td>Ep 2. Power testing was fun, i found my brothe...</td>\n",
              "      <td>😳</td>\n",
              "      <td>[😳 flushed face]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>466 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7eb205ae-1f9f-4284-a464-9134da0f0c0c')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7eb205ae-1f9f-4284-a464-9134da0f0c0c button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7eb205ae-1f9f-4284-a464-9134da0f0c0c');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_523b0477-dc8a-4e37-a1b4-76db72309355\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df2')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_523b0477-dc8a-4e37-a1b4-76db72309355 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df2');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df2",
              "summary": "{\n  \"name\": \"df2\",\n  \"rows\": 466,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 453,\n        \"samples\": [\n          \"me n the boyz went OFFF\\ud83d\\ude2e\\u200d\\ud83d\\udca8\\ud83d\\udd25 #BOYNEXTDOOR #\\ubcf4\\uc774\\ub125\\uc2a4\\ud2b8\\ub3c4\\uc5b4 #BND #\\uba85\\uc7ac\\ud604 #JAEHYUN #\\uc774\\ud55c #LEEHAN #BOYNEXTDOOR_EWF #\\uc5bc\\uc708\\ud30c #EWF_Challenge\",\n          \"may 17th\",\n          \"4 years is actually insane\\ud83d\\ude2d\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"emojis\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 176,\n        \"samples\": [\n          \"\\ud83e\\udd0d \\ud83e\\udd0d \\ud83e\\udd0d \",\n          \"\\ud83d\\udc8c \",\n          \"\\ud83e\\udee1 \\ud83d\\ude2d \\ud83d\\udd25 \"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"emojis list\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extract **LIWC features** by Python - *similar to real LIWC!*"
      ],
      "metadata": {
        "id": "sLSQYkWagy-P"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --quiet contentcoder\n",
        "from contentcoder.ContentCoder import ContentCoder\n",
        "# instantiate coder class\n",
        "cc = ContentCoder(dicFilename='/content/LIWC2007_English100131.dic',\n",
        "                  fileEncoding='utf-8-sig')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_CTNH-zeh39f",
        "outputId": "e5e44760-7ecf-46e3-e1da-171db53ef6f4"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dictionary loaded.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = df[['text']].copy()\n",
        "\n",
        "# get many different features by LIWC dictionary\n",
        "for i in tqdm(range(len(df2))):\n",
        "  text = df2.loc[i,'text']\n",
        "  results = cc.Analyze(text, relativeFreq=True,\n",
        "                             dropPunct=True,\n",
        "                             retainCaptures=False,\n",
        "                             returnTokens=False,\n",
        "                             wildcardMem=True)\n",
        "  for m in results.keys():\n",
        "    df2.loc[i,m] = results[m]\n",
        "\n",
        "df2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 771,
          "referenced_widgets": [
            "3dbe5ab5c5df49bab65a17c3793c0a98",
            "49d6934efad3469081a30e75f7db0009",
            "36cac403431d4306bba0c2e847792ee1",
            "9f64611330e04e89ba257f6292f60eb5",
            "6caf84e1beec465bab2f1c5775554a2b",
            "a84c521367a74ac78cd841b512e12505",
            "4db68f7ca8574a5bb1cdb2a10d4d0032",
            "0f3139326c884b4f914be7627fbbabe6",
            "38965fd20e3044a4949ee97da495d2bf",
            "a8b86c11716c4055812865fe6f00a543",
            "a024c99d89d148f38d6bb0ed1e0720fa"
          ]
        },
        "id": "4Zbaob7Ag2g6",
        "outputId": "8fc0629e-b28c-45d8-b5df-4ce9b59bbc6e"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/466 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3dbe5ab5c5df49bab65a17c3793c0a98"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text        Dic    WC  \\\n",
              "0    sorry to do this lol but i luv it so much also...  81.818182  33.0   \n",
              "1    can’t belieeeeeve it’s been almost a year sinc...  80.487805  41.0   \n",
              "2                                we in australia bitch  75.000000   4.0   \n",
              "3                 ;))))) @Storeroom Vintage love you 😁  33.333333   6.0   \n",
              "4    leave me alone i barely know what im doing but...  65.789474  38.0   \n",
              "..                                                 ...        ...   ...   \n",
              "461  Ep 4. Things were good… until they weren’t @li...  60.606061  33.0   \n",
              "462  back at it again wit the bruddaa🔥 dc: @notorla...  50.000000  10.0   \n",
              "463  Ep 3. Big brother duties👀🔥 @liithosent #liitho...  61.363636  44.0   \n",
              "464  That was clean @jakefromstatefarm ! Go to @kha...  59.375000  32.0   \n",
              "465  Ep 2. Power testing was fun, i found my brothe...  70.454545  44.0   \n",
              "\n",
              "      BigWords   Numbers  AllPunct    Period     Comma  QMark    Exclam  ...  \\\n",
              "0     6.060606  6.060606  0.090909  0.000000  0.000000    0.0  0.000000  ...   \n",
              "1     4.878049  0.000000  0.195122  0.024390  0.024390    0.0  0.000000  ...   \n",
              "2    25.000000  0.000000  0.000000  0.000000  0.000000    0.0  0.000000  ...   \n",
              "3    33.333333  0.000000  1.166667  0.000000  0.000000    0.0  0.000000  ...   \n",
              "4     0.000000  0.000000  0.026316  0.000000  0.000000    0.0  0.000000  ...   \n",
              "..         ...       ...       ...       ...       ...    ...       ...  ...   \n",
              "461  18.181818  3.030303  0.242424  0.060606  0.000000    0.0  0.000000  ...   \n",
              "462  20.000000  0.000000  0.200000  0.000000  0.000000    0.0  0.000000  ...   \n",
              "463  25.000000  4.545455  0.204545  0.045455  0.000000    0.0  0.022727  ...   \n",
              "464  21.875000  6.250000  0.562500  0.093750  0.000000    0.0  0.062500  ...   \n",
              "465  18.181818  2.272727  0.227273  0.090909  0.022727    0.0  0.022727  ...   \n",
              "\n",
              "         work   achieve   leisure      home     money  relig  death    assent  \\\n",
              "0    0.000000  3.030303  0.000000  0.000000  0.000000    0.0    0.0  3.030303   \n",
              "1    2.439024  2.439024  2.439024  2.439024  0.000000    0.0    0.0  0.000000   \n",
              "2    0.000000  0.000000  0.000000  0.000000  0.000000    0.0    0.0  0.000000   \n",
              "3    0.000000  0.000000  0.000000  0.000000  0.000000    0.0    0.0  0.000000   \n",
              "4    2.631579  2.631579  2.631579  0.000000  0.000000    0.0    0.0  0.000000   \n",
              "..        ...       ...       ...       ...       ...    ...    ...       ...   \n",
              "461  0.000000  0.000000  6.060606  0.000000  6.060606    0.0    0.0  0.000000   \n",
              "462  0.000000  0.000000  0.000000  0.000000  0.000000    0.0    0.0  0.000000   \n",
              "463  2.272727  4.545455  4.545455  0.000000  6.818182    0.0    0.0  0.000000   \n",
              "464  0.000000  0.000000  0.000000  3.125000  0.000000    0.0    0.0  0.000000   \n",
              "465  2.272727  2.272727  4.545455  0.000000  4.545455    0.0    0.0  0.000000   \n",
              "\n",
              "     nonfl  filler  \n",
              "0      0.0     0.0  \n",
              "1      0.0     0.0  \n",
              "2      0.0     0.0  \n",
              "3      0.0     0.0  \n",
              "4      0.0     0.0  \n",
              "..     ...     ...  \n",
              "461    0.0     0.0  \n",
              "462    0.0     0.0  \n",
              "463    0.0     0.0  \n",
              "464    0.0     0.0  \n",
              "465    0.0     0.0  \n",
              "\n",
              "[466 rows x 75 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-757d82c0-3404-4299-8f20-4d972a1b3f88\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Dic</th>\n",
              "      <th>WC</th>\n",
              "      <th>BigWords</th>\n",
              "      <th>Numbers</th>\n",
              "      <th>AllPunct</th>\n",
              "      <th>Period</th>\n",
              "      <th>Comma</th>\n",
              "      <th>QMark</th>\n",
              "      <th>Exclam</th>\n",
              "      <th>...</th>\n",
              "      <th>work</th>\n",
              "      <th>achieve</th>\n",
              "      <th>leisure</th>\n",
              "      <th>home</th>\n",
              "      <th>money</th>\n",
              "      <th>relig</th>\n",
              "      <th>death</th>\n",
              "      <th>assent</th>\n",
              "      <th>nonfl</th>\n",
              "      <th>filler</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>sorry to do this lol but i luv it so much also...</td>\n",
              "      <td>81.818182</td>\n",
              "      <td>33.0</td>\n",
              "      <td>6.060606</td>\n",
              "      <td>6.060606</td>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>can’t belieeeeeve it’s been almost a year sinc...</td>\n",
              "      <td>80.487805</td>\n",
              "      <td>41.0</td>\n",
              "      <td>4.878049</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.195122</td>\n",
              "      <td>0.024390</td>\n",
              "      <td>0.024390</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2.439024</td>\n",
              "      <td>2.439024</td>\n",
              "      <td>2.439024</td>\n",
              "      <td>2.439024</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>we in australia bitch</td>\n",
              "      <td>75.000000</td>\n",
              "      <td>4.0</td>\n",
              "      <td>25.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>;))))) @Storeroom Vintage love you 😁</td>\n",
              "      <td>33.333333</td>\n",
              "      <td>6.0</td>\n",
              "      <td>33.333333</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.166667</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>leave me alone i barely know what im doing but...</td>\n",
              "      <td>65.789474</td>\n",
              "      <td>38.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.026316</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>2.631579</td>\n",
              "      <td>2.631579</td>\n",
              "      <td>2.631579</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>461</th>\n",
              "      <td>Ep 4. Things were good… until they weren’t @li...</td>\n",
              "      <td>60.606061</td>\n",
              "      <td>33.0</td>\n",
              "      <td>18.181818</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.242424</td>\n",
              "      <td>0.060606</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>6.060606</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>6.060606</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>462</th>\n",
              "      <td>back at it again wit the bruddaa🔥 dc: @notorla...</td>\n",
              "      <td>50.000000</td>\n",
              "      <td>10.0</td>\n",
              "      <td>20.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>463</th>\n",
              "      <td>Ep 3. Big brother duties👀🔥 @liithosent #liitho...</td>\n",
              "      <td>61.363636</td>\n",
              "      <td>44.0</td>\n",
              "      <td>25.000000</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>0.204545</td>\n",
              "      <td>0.045455</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.022727</td>\n",
              "      <td>...</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>6.818182</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>464</th>\n",
              "      <td>That was clean @jakefromstatefarm ! Go to @kha...</td>\n",
              "      <td>59.375000</td>\n",
              "      <td>32.0</td>\n",
              "      <td>21.875000</td>\n",
              "      <td>6.250000</td>\n",
              "      <td>0.562500</td>\n",
              "      <td>0.093750</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.062500</td>\n",
              "      <td>...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.125000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>465</th>\n",
              "      <td>Ep 2. Power testing was fun, i found my brothe...</td>\n",
              "      <td>70.454545</td>\n",
              "      <td>44.0</td>\n",
              "      <td>18.181818</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>0.227273</td>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.022727</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.022727</td>\n",
              "      <td>...</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>466 rows × 75 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-757d82c0-3404-4299-8f20-4d972a1b3f88')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-757d82c0-3404-4299-8f20-4d972a1b3f88 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-757d82c0-3404-4299-8f20-4d972a1b3f88');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_6f74550f-8915-4702-857e-4d2a9e1c7bba\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df2')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_6f74550f-8915-4702-857e-4d2a9e1c7bba button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df2');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df2"
            }
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# can select a few\n",
        "df2[['text','i','we','body','social','past','future']]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "3BQ1FkD4miJu",
        "outputId": "f6b3cdf4-c356-4e89-ff41-0c719610ec3f"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text          i         we  \\\n",
              "0    sorry to do this lol but i luv it so much also...   3.030303   0.000000   \n",
              "1    can’t belieeeeeve it’s been almost a year sinc...   0.000000   2.439024   \n",
              "2                                we in australia bitch   0.000000  25.000000   \n",
              "3                 ;))))) @Storeroom Vintage love you 😁   0.000000   0.000000   \n",
              "4    leave me alone i barely know what im doing but...  13.157895   0.000000   \n",
              "..                                                 ...        ...        ...   \n",
              "461  Ep 4. Things were good… until they weren’t @li...   0.000000   0.000000   \n",
              "462  back at it again wit the bruddaa🔥 dc: @notorla...   0.000000   0.000000   \n",
              "463  Ep 3. Big brother duties👀🔥 @liithosent #liitho...   0.000000   0.000000   \n",
              "464  That was clean @jakefromstatefarm ! Go to @kha...   0.000000   0.000000   \n",
              "465  Ep 2. Power testing was fun, i found my brothe...   4.545455   0.000000   \n",
              "\n",
              "     body     social      past  future  \n",
              "0     0.0   0.000000  0.000000     0.0  \n",
              "1     0.0  12.195122  4.878049     0.0  \n",
              "2     0.0  25.000000  0.000000     0.0  \n",
              "3     0.0  33.333333  0.000000     0.0  \n",
              "4     0.0   7.894737  0.000000     0.0  \n",
              "..    ...        ...       ...     ...  \n",
              "461   0.0   6.060606  3.030303     0.0  \n",
              "462   0.0   0.000000  0.000000     0.0  \n",
              "463   0.0   2.272727  0.000000     0.0  \n",
              "464   0.0   3.125000  3.125000     0.0  \n",
              "465   0.0   4.545455  6.818182     0.0  \n",
              "\n",
              "[466 rows x 7 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8f6e6a7c-51aa-4b54-b7f1-087075805c3a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>i</th>\n",
              "      <th>we</th>\n",
              "      <th>body</th>\n",
              "      <th>social</th>\n",
              "      <th>past</th>\n",
              "      <th>future</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>sorry to do this lol but i luv it so much also...</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>can’t belieeeeeve it’s been almost a year sinc...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.439024</td>\n",
              "      <td>0.0</td>\n",
              "      <td>12.195122</td>\n",
              "      <td>4.878049</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>we in australia bitch</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>25.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>25.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>;))))) @Storeroom Vintage love you 😁</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>33.333333</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>leave me alone i barely know what im doing but...</td>\n",
              "      <td>13.157895</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.894737</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>461</th>\n",
              "      <td>Ep 4. Things were good… until they weren’t @li...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>6.060606</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>462</th>\n",
              "      <td>back at it again wit the bruddaa🔥 dc: @notorla...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>463</th>\n",
              "      <td>Ep 3. Big brother duties👀🔥 @liithosent #liitho...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>464</th>\n",
              "      <td>That was clean @jakefromstatefarm ! Go to @kha...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>3.125000</td>\n",
              "      <td>3.125000</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>465</th>\n",
              "      <td>Ep 2. Power testing was fun, i found my brothe...</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>6.818182</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>466 rows × 7 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8f6e6a7c-51aa-4b54-b7f1-087075805c3a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-8f6e6a7c-51aa-4b54-b7f1-087075805c3a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-8f6e6a7c-51aa-4b54-b7f1-087075805c3a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df2[['text','i','we','body','social','past','future']]\",\n  \"rows\": 466,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 453,\n        \"samples\": [\n          \"me n the boyz went OFFF\\ud83d\\ude2e\\u200d\\ud83d\\udca8\\ud83d\\udd25 #BOYNEXTDOOR #\\ubcf4\\uc774\\ub125\\uc2a4\\ud2b8\\ub3c4\\uc5b4 #BND #\\uba85\\uc7ac\\ud604 #JAEHYUN #\\uc774\\ud55c #LEEHAN #BOYNEXTDOOR_EWF #\\uc5bc\\uc708\\ud30c #EWF_Challenge\",\n          \"may 17th\",\n          \"4 years is actually insane\\ud83d\\ude2d\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"i\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.603902742477316,\n        \"min\": 0.0,\n        \"max\": 33.33333333333333,\n        \"num_unique_values\": 47,\n        \"samples\": [\n          28.57142857142857,\n          3.8461538461538463,\n          4.3478260869565215\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"we\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 2.7760379020603465,\n        \"min\": 0.0,\n        \"max\": 25.0,\n        \"num_unique_values\": 18,\n        \"samples\": [\n          0.0,\n          2.4390243902439024,\n          14.285714285714285\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"body\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 5.309434337909191,\n        \"min\": 0.0,\n        \"max\": 100.0,\n        \"num_unique_values\": 14,\n        \"samples\": [\n          14.285714285714285,\n          3.571428571428571,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"social\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 10.220411345191716,\n        \"min\": 0.0,\n        \"max\": 100.0,\n        \"num_unique_values\": 60,\n        \"samples\": [\n          0.0,\n          9.67741935483871,\n          2.127659574468085\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"past\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 4.926956673506507,\n        \"min\": 0.0,\n        \"max\": 28.57142857142857,\n        \"num_unique_values\": 43,\n        \"samples\": [\n          3.8461538461538463,\n          8.333333333333332,\n          3.3333333333333335\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"future\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 3.2030589782300103,\n        \"min\": 0.0,\n        \"max\": 50.0,\n        \"num_unique_values\": 22,\n        \"samples\": [\n          0.0,\n          5.88235294117647,\n          12.5\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## The full list of available entries!!!"
      ],
      "metadata": {
        "id": "YpxZXTREUD9V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# this is what is available\n",
        "[i for i in df2.columns]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B-I3mKe3mLJY",
        "outputId": "1884649f-330b-4f74-fdc4-6dea7a2fb03c"
      },
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['text',\n",
              " 'Dic',\n",
              " 'WC',\n",
              " 'BigWords',\n",
              " 'Numbers',\n",
              " 'AllPunct',\n",
              " 'Period',\n",
              " 'Comma',\n",
              " 'QMark',\n",
              " 'Exclam',\n",
              " 'Apostro',\n",
              " 'funct',\n",
              " 'pronoun',\n",
              " 'ppron',\n",
              " 'i',\n",
              " 'we',\n",
              " 'you',\n",
              " 'shehe',\n",
              " 'they',\n",
              " 'ipron',\n",
              " 'article',\n",
              " 'verb',\n",
              " 'auxverb',\n",
              " 'past',\n",
              " 'present',\n",
              " 'future',\n",
              " 'adverb',\n",
              " 'preps',\n",
              " 'conj',\n",
              " 'negate',\n",
              " 'quant',\n",
              " 'number',\n",
              " 'swear',\n",
              " 'social',\n",
              " 'family',\n",
              " 'friend',\n",
              " 'humans',\n",
              " 'affect',\n",
              " 'posemo',\n",
              " 'negemo',\n",
              " 'anx',\n",
              " 'anger',\n",
              " 'sad',\n",
              " 'cogmech',\n",
              " 'insight',\n",
              " 'cause',\n",
              " 'discrep',\n",
              " 'tentat',\n",
              " 'certain',\n",
              " 'inhib',\n",
              " 'incl',\n",
              " 'excl',\n",
              " 'percept',\n",
              " 'see',\n",
              " 'hear',\n",
              " 'feel',\n",
              " 'bio',\n",
              " 'body',\n",
              " 'health',\n",
              " 'sexual',\n",
              " 'ingest',\n",
              " 'relativ',\n",
              " 'motion',\n",
              " 'space',\n",
              " 'time',\n",
              " 'work',\n",
              " 'achieve',\n",
              " 'leisure',\n",
              " 'home',\n",
              " 'money',\n",
              " 'relig',\n",
              " 'death',\n",
              " 'assent',\n",
              " 'nonfl',\n",
              " 'filler']"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Can also use **other** dictionaries"
      ],
      "metadata": {
        "id": "IZharHMJ2VUW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cc2 = ContentCoder(dicFilename='/content/big-two-agency-communion-dictionary.dicx',\n",
        "                  fileEncoding='utf-8-sig')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9ff-s9aE2kyQ",
        "outputId": "f696f6e7-6e8c-4dde-88b2-7c3b9b7435bd"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dictionary loaded.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = df[['text']].copy()\n",
        "\n",
        "# get many different features by LIWC dictionary\n",
        "for i in tqdm(range(len(df2))):\n",
        "  text = df2.loc[i,'text']\n",
        "  results = cc2.Analyze(text, relativeFreq=True,\n",
        "                             dropPunct=True,\n",
        "                             retainCaptures=False,\n",
        "                             returnTokens=False,\n",
        "                             wildcardMem=True)\n",
        "  for m in results.keys():\n",
        "    df2.loc[i,m] = results[m]\n",
        "\n",
        "df2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 771,
          "referenced_widgets": [
            "6e15bf11274e4421811c5baab61ebd9c",
            "ac9220e42a5f4760bd24ea1b0d70feea",
            "a65cb41b5598491093a45659ff3ff06b",
            "3a881f38e4494de3be92b7a6eda297b4",
            "1ad69d46f9ac4c0eb95abaebe92c81a3",
            "7a5521828ff5430c8f6f4faff1ffa7d7",
            "91bb71fc85a647bbbf1db62885ec6dc4",
            "7c35c13a14b34ef0a8515bfa08b61a5b",
            "0d14e097bcf6478d9b3bb310e64032be",
            "1cad13fa820b44318f358a11f7c8f6b3",
            "51d6dea2265646a69cab44f38ccc643c"
          ]
        },
        "id": "vP8F-Gvj2g_s",
        "outputId": "c1659acc-cf79-47b8-ff3b-e8e270ac5c84"
      },
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/466 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6e15bf11274e4421811c5baab61ebd9c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text        Dic    WC  \\\n",
              "0    sorry to do this lol but i luv it so much also...   3.030303  33.0   \n",
              "1    can’t belieeeeeve it’s been almost a year sinc...  14.634146  41.0   \n",
              "2                                we in australia bitch  25.000000   4.0   \n",
              "3                 ;))))) @Storeroom Vintage love you 😁  33.333333   6.0   \n",
              "4    leave me alone i barely know what im doing but...  13.157895  38.0   \n",
              "..                                                 ...        ...   ...   \n",
              "461  Ep 4. Things were good… until they weren’t @li...   3.030303  33.0   \n",
              "462  back at it again wit the bruddaa🔥 dc: @notorla...  10.000000  10.0   \n",
              "463  Ep 3. Big brother duties👀🔥 @liithosent #liitho...   0.000000  44.0   \n",
              "464  That was clean @jakefromstatefarm ! Go to @kha...   0.000000  32.0   \n",
              "465  Ep 2. Power testing was fun, i found my brothe...   2.272727  44.0   \n",
              "\n",
              "      BigWords   Numbers  AllPunct    Period     Comma  QMark    Exclam  \\\n",
              "0     6.060606  6.060606  0.090909  0.000000  0.000000    0.0  0.000000   \n",
              "1     4.878049  0.000000  0.195122  0.024390  0.024390    0.0  0.000000   \n",
              "2    25.000000  0.000000  0.000000  0.000000  0.000000    0.0  0.000000   \n",
              "3    33.333333  0.000000  1.166667  0.000000  0.000000    0.0  0.000000   \n",
              "4     0.000000  0.000000  0.026316  0.000000  0.000000    0.0  0.000000   \n",
              "..         ...       ...       ...       ...       ...    ...       ...   \n",
              "461  18.181818  3.030303  0.242424  0.060606  0.000000    0.0  0.000000   \n",
              "462  20.000000  0.000000  0.200000  0.000000  0.000000    0.0  0.000000   \n",
              "463  25.000000  4.545455  0.204545  0.045455  0.000000    0.0  0.022727   \n",
              "464  21.875000  6.250000  0.562500  0.093750  0.000000    0.0  0.062500   \n",
              "465  18.181818  2.272727  0.227273  0.090909  0.022727    0.0  0.022727   \n",
              "\n",
              "      Apostro     Agency  Communion  \n",
              "0    0.000000   3.030303   0.000000  \n",
              "1    0.073171   4.878049   9.756098  \n",
              "2    0.000000   0.000000  25.000000  \n",
              "3    0.000000  16.666667  16.666667  \n",
              "4    0.000000  10.526316   2.631579  \n",
              "..        ...        ...        ...  \n",
              "461  0.030303   3.030303   0.000000  \n",
              "462  0.000000  10.000000   0.000000  \n",
              "463  0.000000   0.000000   0.000000  \n",
              "464  0.031250   0.000000   0.000000  \n",
              "465  0.000000   2.272727   0.000000  \n",
              "\n",
              "[466 rows x 13 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-df3c21e1-60e4-4183-8bb7-12328b2fad5a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Dic</th>\n",
              "      <th>WC</th>\n",
              "      <th>BigWords</th>\n",
              "      <th>Numbers</th>\n",
              "      <th>AllPunct</th>\n",
              "      <th>Period</th>\n",
              "      <th>Comma</th>\n",
              "      <th>QMark</th>\n",
              "      <th>Exclam</th>\n",
              "      <th>Apostro</th>\n",
              "      <th>Agency</th>\n",
              "      <th>Communion</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>sorry to do this lol but i luv it so much also...</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>33.0</td>\n",
              "      <td>6.060606</td>\n",
              "      <td>6.060606</td>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>can’t belieeeeeve it’s been almost a year sinc...</td>\n",
              "      <td>14.634146</td>\n",
              "      <td>41.0</td>\n",
              "      <td>4.878049</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.195122</td>\n",
              "      <td>0.024390</td>\n",
              "      <td>0.024390</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.073171</td>\n",
              "      <td>4.878049</td>\n",
              "      <td>9.756098</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>we in australia bitch</td>\n",
              "      <td>25.000000</td>\n",
              "      <td>4.0</td>\n",
              "      <td>25.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>25.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>;))))) @Storeroom Vintage love you 😁</td>\n",
              "      <td>33.333333</td>\n",
              "      <td>6.0</td>\n",
              "      <td>33.333333</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>1.166667</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>16.666667</td>\n",
              "      <td>16.666667</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>leave me alone i barely know what im doing but...</td>\n",
              "      <td>13.157895</td>\n",
              "      <td>38.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.026316</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>10.526316</td>\n",
              "      <td>2.631579</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>461</th>\n",
              "      <td>Ep 4. Things were good… until they weren’t @li...</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>33.0</td>\n",
              "      <td>18.181818</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.242424</td>\n",
              "      <td>0.060606</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.030303</td>\n",
              "      <td>3.030303</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>462</th>\n",
              "      <td>back at it again wit the bruddaa🔥 dc: @notorla...</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>10.0</td>\n",
              "      <td>20.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.200000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>10.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>463</th>\n",
              "      <td>Ep 3. Big brother duties👀🔥 @liithosent #liitho...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>44.0</td>\n",
              "      <td>25.000000</td>\n",
              "      <td>4.545455</td>\n",
              "      <td>0.204545</td>\n",
              "      <td>0.045455</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.022727</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>464</th>\n",
              "      <td>That was clean @jakefromstatefarm ! Go to @kha...</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>32.0</td>\n",
              "      <td>21.875000</td>\n",
              "      <td>6.250000</td>\n",
              "      <td>0.562500</td>\n",
              "      <td>0.093750</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.062500</td>\n",
              "      <td>0.031250</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>465</th>\n",
              "      <td>Ep 2. Power testing was fun, i found my brothe...</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>44.0</td>\n",
              "      <td>18.181818</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>0.227273</td>\n",
              "      <td>0.090909</td>\n",
              "      <td>0.022727</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.022727</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2.272727</td>\n",
              "      <td>0.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>466 rows × 13 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-df3c21e1-60e4-4183-8bb7-12328b2fad5a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-df3c21e1-60e4-4183-8bb7-12328b2fad5a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-df3c21e1-60e4-4183-8bb7-12328b2fad5a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_2471bad9-edaf-40c2-b5ef-b281bbe94845\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df2')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_2471bad9-edaf-40c2-b5ef-b281bbe94845 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df2');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df2",
              "summary": "{\n  \"name\": \"df2\",\n  \"rows\": 466,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 453,\n        \"samples\": [\n          \"me n the boyz went OFFF\\ud83d\\ude2e\\u200d\\ud83d\\udca8\\ud83d\\udd25 #BOYNEXTDOOR #\\ubcf4\\uc774\\ub125\\uc2a4\\ud2b8\\ub3c4\\uc5b4 #BND #\\uba85\\uc7ac\\ud604 #JAEHYUN #\\uc774\\ud55c #LEEHAN #BOYNEXTDOOR_EWF #\\uc5bc\\uc708\\ud30c #EWF_Challenge\",\n          \"may 17th\",\n          \"4 years is actually insane\\ud83d\\ude2d\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Dic\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 9.17849182340479,\n        \"min\": 0.0,\n        \"max\": 50.0,\n        \"num_unique_values\": 61,\n        \"samples\": [\n          3.0303030303030303,\n          0.0,\n          3.125\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"WC\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 11.640980736487307,\n        \"min\": 0.0,\n        \"max\": 104.0,\n        \"num_unique_values\": 49,\n        \"samples\": [\n          9.0,\n          79.0,\n          48.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"BigWords\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 18.94455463593021,\n        \"min\": 0.0,\n        \"max\": 100.0,\n        \"num_unique_values\": 88,\n        \"samples\": [\n          30.909090909090896,\n          6.0606060606060606,\n          18.181818181818183\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Numbers\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.270113833636813,\n        \"min\": 0.0,\n        \"max\": 50.0,\n        \"num_unique_values\": 40,\n        \"samples\": [\n          7.6923076923076925,\n          4.651162790697675,\n          7.692307692307692\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"AllPunct\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.24907085088304282,\n        \"min\": 0.0,\n        \"max\": 1.2,\n        \"num_unique_values\": 100,\n        \"samples\": [\n          0.4411764705882353,\n          0.35294117647058826,\n          0.42105263157894735\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Period\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.0352451406784415,\n        \"min\": 0.0,\n        \"max\": 0.5,\n        \"num_unique_values\": 35,\n        \"samples\": [\n          0.01818181818181818,\n          0.07692307692307693,\n          0.034482758620689655\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Comma\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.014722322246574833,\n        \"min\": 0.0,\n        \"max\": 0.14285714285714285,\n        \"num_unique_values\": 22,\n        \"samples\": [\n          0.0,\n          0.04807692307692308,\n          0.058823529411764705\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"QMark\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.037695750547248955,\n        \"min\": 0.0,\n        \"max\": 0.5,\n        \"num_unique_values\": 14,\n        \"samples\": [\n          0.04,\n          0.125,\n          0.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Exclam\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07114656511895948,\n        \"min\": 0.0,\n        \"max\": 1.0,\n        \"num_unique_values\": 37,\n        \"samples\": [\n          0.06666666666666667,\n          0.16666666666666666,\n          0.02564102564102564\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Apostro\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.031016134555345107,\n        \"min\": 0.0,\n        \"max\": 0.3333333333333333,\n        \"num_unique_values\": 27,\n        \"samples\": [\n          0.11764705882352941,\n          0.038461538461538464,\n          0.3333333333333333\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Agency\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.324520115653935,\n        \"min\": 0.0,\n        \"max\": 50.0,\n        \"num_unique_values\": 47,\n        \"samples\": [\n          6.666666666666667,\n          1.8181818181818181,\n          11.11111111111111\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"Communion\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.383640098315793,\n        \"min\": 0.0,\n        \"max\": 50.0,\n        \"num_unique_values\": 40,\n        \"samples\": [\n          10.0,\n          11.627906976744187,\n          14.285714285714285\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Extract **sentiment**, **emotions**, and **agency**"
      ],
      "metadata": {
        "id": "A4uWrtRA5Mqy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# https://huggingface.co/cardiffnlp/twitter-roberta-base-sentiment-latest\n",
        "# https://huggingface.co/cardiffnlp/twitter-roberta-large-emotion-latest\n",
        "# https://pypi.org/project/bertagent/\n",
        "#\n",
        "from transformers import pipeline\n",
        "model_path = f\"cardiffnlp/twitter-roberta-base-sentiment-latest\"\n",
        "sentiment_task = pipeline(\"sentiment-analysis\",\n",
        "                          model=model_path,\n",
        "                          tokenizer=model_path, top_k=None)\n",
        "emotion_task = pipeline('text-classification',\n",
        "                        model=\"cardiffnlp/twitter-roberta-large-emotion-latest\",\n",
        "                        top_k=None)\n",
        "agency_task = pipeline('text-classification',\n",
        "                        model=\"EnchantedStardust/bertagent-best\",\n",
        "                        top_k=None)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 256,
          "referenced_widgets": [
            "a1a24db84f834a5fb09a2f68745283fc",
            "abf6cf8fb40f4e709add0ef137d94250",
            "6fadd79bf93d4fa0b7b74bbcdb90e5ee",
            "cb9637fafda3444eacd9e30760d96643",
            "a9cff0f706a94680abd0024edae2cd89",
            "1879d56f9fa645eaa491d1d1d553737a",
            "88d8ecc6f2f4459590d7410f065f802e",
            "f1a0cd3e14364b66be72973abb225194",
            "a410d6b24c814e79b25055182ba4ff2f",
            "60caa1b60cd940908ac72c4f7f409a92",
            "88c5487b9c794716a8c1520954d04c4c",
            "1e70a444795649e9826833f705a7f31e",
            "ef207f40d03545869fb6f66d313e7680",
            "1d8e81c4082647a59f23ba1487ce4a3f",
            "b0cc9541bacb441b9d3ef8092cc65488",
            "2acd0879e55d45f0bffcbad69c9245e4",
            "4ed80b03cad7415198c163697a87c71e",
            "0a5ea60bc77340f58f3a9be2b76ecd85",
            "cd6afa26b5244661aef57aa403be3cd0",
            "10bf7a1ecb8f497abeef835a5196cc98",
            "845282cba6f242fe982e368e6dc0b0df",
            "7610658073804abfb608cd693b900e4d",
            "f8fe5e57cea64311b7a020c1f4622e7c",
            "1f07ce83319a4e6a9df9f7ff0ea2c8b9",
            "951bc75b70ab4887989a86633bef9d1f",
            "6233ab84732d498bbb4c108ba18623dd",
            "cb49f8ec4fc145d4871d8a2d898c07c2",
            "6d91b1b00721453294d7cde996258c08",
            "ebfd23f041f848139c5b7bf91df56b65",
            "aff2664fbe994414a1fba8e393a2cb1f",
            "d71a914345dd46138d460eafd98417aa",
            "1ad54ed15ead4badb03194de644d04a2",
            "49ddf846a08947fe9da092dfce2f20ed"
          ]
        },
        "id": "NrN7Owqi-DJq",
        "outputId": "4669cd7b-734f-40d0-85a9-d881342a9633"
      },
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading weights:   0%|          | 0/201 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a1a24db84f834a5fb09a2f68745283fc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[transformers] \u001b[1mRobertaForSequenceClassification LOAD REPORT\u001b[0m from: cardiffnlp/twitter-roberta-base-sentiment-latest\n",
            "Key                         | Status     |  | \n",
            "----------------------------+------------+--+-\n",
            "roberta.pooler.dense.bias   | UNEXPECTED |  | \n",
            "roberta.pooler.dense.weight | UNEXPECTED |  | \n",
            "\n",
            "Notes:\n",
            "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading weights:   0%|          | 0/393 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1e70a444795649e9826833f705a7f31e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading weights:   0%|          | 0/201 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f8fe5e57cea64311b7a020c1f4622e7c"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df2 = df[['text']].copy()\n",
        "\n",
        "for i in tqdm(range(len(df2))):\n",
        "  text = df2.loc[i,'text']\n",
        "\n",
        "  predictions = sentiment_task(text)[0]\n",
        "  for j in range(len(predictions)):\n",
        "    df2.loc[i, predictions[j]['label']] = round(predictions[j]['score'],4)\n",
        "\n",
        "  predictions = emotion_task(text)[0]\n",
        "  for j in range(len(predictions)):\n",
        "    df2.loc[i, predictions[j]['label']] = round(predictions[j]['score'],4)\n",
        "\n",
        "  predictions = agency_task(text)[0]\n",
        "  for j in range(len(predictions)):\n",
        "    df2.loc[i, predictions[j]['label']] = round(predictions[j]['score'],4)\n",
        "\n",
        "df2.rename(columns={'LABEL_0':'agency'}, inplace=True)\n",
        "df2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 771,
          "referenced_widgets": [
            "0f13edcd0dfd417696442b29c37ae54b",
            "83916cf10e32489f99f226acde7fcfc7",
            "4e6e177af5ee42d8aa8904e8de4ebada",
            "47e41ddedf4a48689fedb0218fcba565",
            "39cfe94aac7f455490bce279492bd9fe",
            "5bb5a8fbe711462db293e48722bb54fe",
            "e8939263ba0c445994dfbb7cc3bf2a8b",
            "e6316af994594a74b8a2dd42d6dd0922",
            "b8a27b4eab0148728611b2531a0b63be",
            "5ddb776d8f014408ab46b4348eaab3aa",
            "94bd66260bac475b9d38b0180312ba50"
          ]
        },
        "id": "hDFD3Qxm6pMp",
        "outputId": "973caf55-09d1-4c7c-ca3c-e2f445874c6b"
      },
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/466 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f13edcd0dfd417696442b29c37ae54b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  text  positive  neutral  \\\n",
              "0    sorry to do this lol but i luv it so much also...    0.9760   0.0161   \n",
              "1    can’t belieeeeeve it’s been almost a year sinc...    0.9882   0.0101   \n",
              "2                                we in australia bitch    0.0578   0.5277   \n",
              "3                 ;))))) @Storeroom Vintage love you 😁    0.9839   0.0134   \n",
              "4    leave me alone i barely know what im doing but...    0.8629   0.1174   \n",
              "..                                                 ...       ...      ...   \n",
              "461  Ep 4. Things were good… until they weren’t @li...    0.8574   0.1365   \n",
              "462  back at it again wit the bruddaa🔥 dc: @notorla...    0.9005   0.0966   \n",
              "463  Ep 3. Big brother duties👀🔥 @liithosent #liitho...    0.7402   0.2577   \n",
              "464  That was clean @jakefromstatefarm ! Go to @kha...    0.9421   0.0552   \n",
              "465  Ep 2. Power testing was fun, i found my brothe...    0.4851   0.3863   \n",
              "\n",
              "     negative     joy    love  optimism   trust  sadness  anticipation  \\\n",
              "0      0.0079  0.9924  0.9890    0.5543  0.0832   0.0254        0.0216   \n",
              "1      0.0017  0.9975  0.9755    0.7357  0.1147   0.0298        0.0304   \n",
              "2      0.4145  0.0119  0.0066    0.0115  0.0074   0.0632        0.0228   \n",
              "3      0.0026  0.9820  0.9910    0.5200  0.0608   0.0289        0.0178   \n",
              "4      0.0197  0.9952  0.9728    0.7326  0.0561   0.0209        0.0239   \n",
              "..        ...     ...     ...       ...     ...      ...           ...   \n",
              "461    0.0061  0.5656  0.0064    0.3134  0.0143   0.4026        0.4983   \n",
              "462    0.0029  0.4806  0.0046    0.0299  0.0075   0.0519        0.2860   \n",
              "463    0.0021  0.9820  0.0711    0.5783  0.0446   0.0093        0.6842   \n",
              "464    0.0027  0.9730  0.0560    0.7447  0.0981   0.0059        0.7306   \n",
              "465    0.1287  0.9748  0.0106    0.1112  0.0100   0.7431        0.2674   \n",
              "\n",
              "     surprise   anger  disgust  pessimism    fear  agency  \n",
              "0      0.0190  0.0166   0.0135     0.0133  0.0083 -0.0137  \n",
              "1      0.0462  0.0052   0.0044     0.0094  0.0099 -0.0649  \n",
              "2      0.0129  0.9914   0.9186     0.0218  0.0094 -0.0753  \n",
              "3      0.0111  0.0200   0.0222     0.0164  0.0096  0.0140  \n",
              "4      0.0154  0.0049   0.0063     0.0105  0.0130 -0.1881  \n",
              "..        ...     ...      ...        ...     ...     ...  \n",
              "461    0.0245  0.0166   0.0191     0.0263  0.0032  0.0321  \n",
              "462    0.0230  0.0624   0.0363     0.0105  0.0023 -0.0338  \n",
              "463    0.0636  0.0046   0.0030     0.0050  0.0059  0.2210  \n",
              "464    0.0561  0.0074   0.0066     0.0048  0.0032  0.1483  \n",
              "465    0.1120  0.0196   0.0909     0.0954  0.0334  0.2095  \n",
              "\n",
              "[466 rows x 16 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2d4a81cc-8351-4492-8a81-e2093d90524a\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>positive</th>\n",
              "      <th>neutral</th>\n",
              "      <th>negative</th>\n",
              "      <th>joy</th>\n",
              "      <th>love</th>\n",
              "      <th>optimism</th>\n",
              "      <th>trust</th>\n",
              "      <th>sadness</th>\n",
              "      <th>anticipation</th>\n",
              "      <th>surprise</th>\n",
              "      <th>anger</th>\n",
              "      <th>disgust</th>\n",
              "      <th>pessimism</th>\n",
              "      <th>fear</th>\n",
              "      <th>agency</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>sorry to do this lol but i luv it so much also...</td>\n",
              "      <td>0.9760</td>\n",
              "      <td>0.0161</td>\n",
              "      <td>0.0079</td>\n",
              "      <td>0.9924</td>\n",
              "      <td>0.9890</td>\n",
              "      <td>0.5543</td>\n",
              "      <td>0.0832</td>\n",
              "      <td>0.0254</td>\n",
              "      <td>0.0216</td>\n",
              "      <td>0.0190</td>\n",
              "      <td>0.0166</td>\n",
              "      <td>0.0135</td>\n",
              "      <td>0.0133</td>\n",
              "      <td>0.0083</td>\n",
              "      <td>-0.0137</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>can’t belieeeeeve it’s been almost a year sinc...</td>\n",
              "      <td>0.9882</td>\n",
              "      <td>0.0101</td>\n",
              "      <td>0.0017</td>\n",
              "      <td>0.9975</td>\n",
              "      <td>0.9755</td>\n",
              "      <td>0.7357</td>\n",
              "      <td>0.1147</td>\n",
              "      <td>0.0298</td>\n",
              "      <td>0.0304</td>\n",
              "      <td>0.0462</td>\n",
              "      <td>0.0052</td>\n",
              "      <td>0.0044</td>\n",
              "      <td>0.0094</td>\n",
              "      <td>0.0099</td>\n",
              "      <td>-0.0649</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>we in australia bitch</td>\n",
              "      <td>0.0578</td>\n",
              "      <td>0.5277</td>\n",
              "      <td>0.4145</td>\n",
              "      <td>0.0119</td>\n",
              "      <td>0.0066</td>\n",
              "      <td>0.0115</td>\n",
              "      <td>0.0074</td>\n",
              "      <td>0.0632</td>\n",
              "      <td>0.0228</td>\n",
              "      <td>0.0129</td>\n",
              "      <td>0.9914</td>\n",
              "      <td>0.9186</td>\n",
              "      <td>0.0218</td>\n",
              "      <td>0.0094</td>\n",
              "      <td>-0.0753</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>;))))) @Storeroom Vintage love you 😁</td>\n",
              "      <td>0.9839</td>\n",
              "      <td>0.0134</td>\n",
              "      <td>0.0026</td>\n",
              "      <td>0.9820</td>\n",
              "      <td>0.9910</td>\n",
              "      <td>0.5200</td>\n",
              "      <td>0.0608</td>\n",
              "      <td>0.0289</td>\n",
              "      <td>0.0178</td>\n",
              "      <td>0.0111</td>\n",
              "      <td>0.0200</td>\n",
              "      <td>0.0222</td>\n",
              "      <td>0.0164</td>\n",
              "      <td>0.0096</td>\n",
              "      <td>0.0140</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>leave me alone i barely know what im doing but...</td>\n",
              "      <td>0.8629</td>\n",
              "      <td>0.1174</td>\n",
              "      <td>0.0197</td>\n",
              "      <td>0.9952</td>\n",
              "      <td>0.9728</td>\n",
              "      <td>0.7326</td>\n",
              "      <td>0.0561</td>\n",
              "      <td>0.0209</td>\n",
              "      <td>0.0239</td>\n",
              "      <td>0.0154</td>\n",
              "      <td>0.0049</td>\n",
              "      <td>0.0063</td>\n",
              "      <td>0.0105</td>\n",
              "      <td>0.0130</td>\n",
              "      <td>-0.1881</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>461</th>\n",
              "      <td>Ep 4. Things were good… until they weren’t @li...</td>\n",
              "      <td>0.8574</td>\n",
              "      <td>0.1365</td>\n",
              "      <td>0.0061</td>\n",
              "      <td>0.5656</td>\n",
              "      <td>0.0064</td>\n",
              "      <td>0.3134</td>\n",
              "      <td>0.0143</td>\n",
              "      <td>0.4026</td>\n",
              "      <td>0.4983</td>\n",
              "      <td>0.0245</td>\n",
              "      <td>0.0166</td>\n",
              "      <td>0.0191</td>\n",
              "      <td>0.0263</td>\n",
              "      <td>0.0032</td>\n",
              "      <td>0.0321</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>462</th>\n",
              "      <td>back at it again wit the bruddaa🔥 dc: @notorla...</td>\n",
              "      <td>0.9005</td>\n",
              "      <td>0.0966</td>\n",
              "      <td>0.0029</td>\n",
              "      <td>0.4806</td>\n",
              "      <td>0.0046</td>\n",
              "      <td>0.0299</td>\n",
              "      <td>0.0075</td>\n",
              "      <td>0.0519</td>\n",
              "      <td>0.2860</td>\n",
              "      <td>0.0230</td>\n",
              "      <td>0.0624</td>\n",
              "      <td>0.0363</td>\n",
              "      <td>0.0105</td>\n",
              "      <td>0.0023</td>\n",
              "      <td>-0.0338</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>463</th>\n",
              "      <td>Ep 3. Big brother duties👀🔥 @liithosent #liitho...</td>\n",
              "      <td>0.7402</td>\n",
              "      <td>0.2577</td>\n",
              "      <td>0.0021</td>\n",
              "      <td>0.9820</td>\n",
              "      <td>0.0711</td>\n",
              "      <td>0.5783</td>\n",
              "      <td>0.0446</td>\n",
              "      <td>0.0093</td>\n",
              "      <td>0.6842</td>\n",
              "      <td>0.0636</td>\n",
              "      <td>0.0046</td>\n",
              "      <td>0.0030</td>\n",
              "      <td>0.0050</td>\n",
              "      <td>0.0059</td>\n",
              "      <td>0.2210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>464</th>\n",
              "      <td>That was clean @jakefromstatefarm ! Go to @kha...</td>\n",
              "      <td>0.9421</td>\n",
              "      <td>0.0552</td>\n",
              "      <td>0.0027</td>\n",
              "      <td>0.9730</td>\n",
              "      <td>0.0560</td>\n",
              "      <td>0.7447</td>\n",
              "      <td>0.0981</td>\n",
              "      <td>0.0059</td>\n",
              "      <td>0.7306</td>\n",
              "      <td>0.0561</td>\n",
              "      <td>0.0074</td>\n",
              "      <td>0.0066</td>\n",
              "      <td>0.0048</td>\n",
              "      <td>0.0032</td>\n",
              "      <td>0.1483</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>465</th>\n",
              "      <td>Ep 2. Power testing was fun, i found my brothe...</td>\n",
              "      <td>0.4851</td>\n",
              "      <td>0.3863</td>\n",
              "      <td>0.1287</td>\n",
              "      <td>0.9748</td>\n",
              "      <td>0.0106</td>\n",
              "      <td>0.1112</td>\n",
              "      <td>0.0100</td>\n",
              "      <td>0.7431</td>\n",
              "      <td>0.2674</td>\n",
              "      <td>0.1120</td>\n",
              "      <td>0.0196</td>\n",
              "      <td>0.0909</td>\n",
              "      <td>0.0954</td>\n",
              "      <td>0.0334</td>\n",
              "      <td>0.2095</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>466 rows × 16 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2d4a81cc-8351-4492-8a81-e2093d90524a')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2d4a81cc-8351-4492-8a81-e2093d90524a button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2d4a81cc-8351-4492-8a81-e2093d90524a');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "  <div id=\"id_393d5321-b5d5-4f80-9167-f2c62f9efe77\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df2')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_393d5321-b5d5-4f80-9167-f2c62f9efe77 button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('df2');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "df2",
              "summary": "{\n  \"name\": \"df2\",\n  \"rows\": 466,\n  \"fields\": [\n    {\n      \"column\": \"text\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 453,\n        \"samples\": [\n          \"me n the boyz went OFFF\\ud83d\\ude2e\\u200d\\ud83d\\udca8\\ud83d\\udd25 #BOYNEXTDOOR #\\ubcf4\\uc774\\ub125\\uc2a4\\ud2b8\\ub3c4\\uc5b4 #BND #\\uba85\\uc7ac\\ud604 #JAEHYUN #\\uc774\\ud55c #LEEHAN #BOYNEXTDOOR_EWF #\\uc5bc\\uc708\\ud30c #EWF_Challenge\",\n          \"may 17th\",\n          \"4 years is actually insane\\ud83d\\ude2d\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"positive\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.3702933146276174,\n        \"min\": 0.0081,\n        \"max\": 0.9916,\n        \"num_unique_values\": 432,\n        \"samples\": [\n          0.9829,\n          0.0221,\n          0.1119\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"neutral\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.3004305206604166,\n        \"min\": 0.0063,\n        \"max\": 0.9491,\n        \"num_unique_values\": 435,\n        \"samples\": [\n          0.1373,\n          0.0436,\n          0.0126\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"negative\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.23582240076901628,\n        \"min\": 0.0012,\n        \"max\": 0.9283,\n        \"num_unique_values\": 309,\n        \"samples\": [\n          0.0207,\n          0.0038,\n          0.0795\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"joy\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.39881292481603814,\n        \"min\": 0.0022,\n        \"max\": 0.9989,\n        \"num_unique_values\": 402,\n        \"samples\": [\n          0.1025,\n          0.0214,\n          0.2968\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"love\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.270503439923469,\n        \"min\": 0.0012,\n        \"max\": 0.9942,\n        \"num_unique_values\": 349,\n        \"samples\": [\n          0.0082,\n          0.1344,\n          0.2983\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"optimism\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.30868907268180307,\n        \"min\": 0.0019,\n        \"max\": 0.9935,\n        \"num_unique_values\": 426,\n        \"samples\": [\n          0.7837,\n          0.0197,\n          0.5176\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"trust\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.07517560761015535,\n        \"min\": 0.0017,\n        \"max\": 0.6868,\n        \"num_unique_values\": 351,\n        \"samples\": [\n          0.0509,\n          0.0539,\n          0.0433\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"sadness\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.31820524264645633,\n        \"min\": 0.003,\n        \"max\": 0.9972,\n        \"num_unique_values\": 386,\n        \"samples\": [\n          0.4103,\n          0.19,\n          0.6816\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"anticipation\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.27185145115348264,\n        \"min\": 0.0101,\n        \"max\": 0.9834,\n        \"num_unique_values\": 432,\n        \"samples\": [\n          0.123,\n          0.0496,\n          0.5742\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"surprise\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.13309789359588745,\n        \"min\": 0.003,\n        \"max\": 0.9235,\n        \"num_unique_values\": 382,\n        \"samples\": [\n          0.1152,\n          0.0883,\n          0.0426\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"anger\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.20425302011383287,\n        \"min\": 0.0019,\n        \"max\": 0.9965,\n        \"num_unique_values\": 292,\n        \"samples\": [\n          0.0024,\n          0.9874,\n          0.0032\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"disgust\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.2036086764176741,\n        \"min\": 0.0022,\n        \"max\": 0.9867,\n        \"num_unique_values\": 313,\n        \"samples\": [\n          0.0077,\n          0.0105,\n          0.0148\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pessimism\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.12160806235071486,\n        \"min\": 0.0031,\n        \"max\": 0.8293,\n        \"num_unique_values\": 268,\n        \"samples\": [\n          0.8293,\n          0.188,\n          0.0208\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"fear\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.15730082931717781,\n        \"min\": 0.0013,\n        \"max\": 0.9977,\n        \"num_unique_values\": 259,\n        \"samples\": [\n          0.9969,\n          0.0187,\n          0.0102\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"agency\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.16303111496553577,\n        \"min\": -0.6805,\n        \"max\": 0.4434,\n        \"num_unique_values\": 431,\n        \"samples\": [\n          0.2043,\n          -0.1692,\n          -0.0061\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Translate** (but seems not to work for free anymore)"
      ],
      "metadata": {
        "id": "DomNXHysXV3d"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --quiet deep-translator\n",
        "from deep_translator import GoogleTranslator\n",
        "gt = GoogleTranslator(source='auto', target='en')\n",
        "import time"
      ],
      "metadata": {
        "id": "gracObUbY_1h"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# translate\n",
        "for i in tqdm(range(len(df))):\n",
        "  text = df.loc[i,'text']\n",
        "  df.loc[i,'translated'] = gt.translate(text)\n",
        "\n",
        "df.dropna(inplace=True) # this simplifies things later on!!!\n",
        "df.reset_index(drop=True, inplace=True)\n",
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408,
          "referenced_widgets": [
            "1cb92e8ef24e4acab7d2517f386188cf",
            "1d8291451ab4467cb6d5d34611375fc6",
            "956d0c9c40584048a16e4a312bf95fde",
            "70cfc3f0c5dd45dd84ee9fea8b6a4403",
            "ba2ce674a696451ab255f91e771a1c4b",
            "180d14b9b32447058433816ede975cdd",
            "c74a23f21fdb46e7aa458e71a8a7a4e0",
            "c5b9f6e5fffc42d5a6fb0a0087ad3bb2",
            "f9328c742b2d42da988c710485e6e7dd",
            "2b9e612435a44017a8630fbcb9f2adf3",
            "1207a203bf484e918b9fae57dd34ba49"
          ]
        },
        "id": "-rYqLAeDXbXx",
        "outputId": "c69093a3-c926-433f-964a-cbf72ffb6785"
      },
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/466 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1cb92e8ef24e4acab7d2517f386188cf"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "TooManyRequests",
          "evalue": "Server Error: You made too many requests to the server.According to google, you are allowed to make 5 requests per secondand up to 200k requests per day. You can wait and try again later oryou can try the translate_batch function",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTooManyRequests\u001b[0m                           Traceback (most recent call last)",
            "\u001b[0;32m/tmp/ipykernel_1400/4129980257.py\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m   \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m   \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'translated'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranslate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# this simplifies things later on!!!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.13/dist-packages/deep_translator/google.py\u001b[0m in \u001b[0;36mtranslate\u001b[0;34m(self, text, **kwargs)\u001b[0m\n\u001b[1;32m     69\u001b[0m             )\n\u001b[1;32m     70\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m429\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mTooManyRequests\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mrequest_failed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus_code\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTooManyRequests\u001b[0m: Server Error: You made too many requests to the server.According to google, you are allowed to make 5 requests per secondand up to 200k requests per day. You can wait and try again later oryou can try the translate_batch function"
          ]
        }
      ]
    }
  ]
}