Skip to content

Instantly share code, notes, and snippets.

@alonsosilvaallende

alonsosilvaallende/understanding_next_token_prediction.ipynb

Created January 21, 2026 08:46
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save alonsosilvaallende/f1daa7b0debd26ea6830b48cef3b1282 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save alonsosilvaallende/f1daa7b0debd26ea6830b48cef3b1282 to your computer and use it in GitHub Desktop.
Download ZIP
Understanding_Next_Token_Prediction.ipynb
Display the source blob
Display the rendered blob
Raw
understanding_next_token_prediction.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"mount_file_id": "1E_vbYj3d0YGNttOByVnXpslqWlLVXObB",
"authorship_tag": "ABX9TyOXLEhJa0XWp+lc7F9n/TnL",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"3b8586dace7a4005a07ee8c67f80c1f4": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_c3f50786ba7c45f1b586a3b854b1aaac",
"IPY_MODEL_70a4a5ad30b043b6b2d0b0761843dd37",
"IPY_MODEL_e9b5b37bf4064247bf5eff86e294284a"
],
"layout": "IPY_MODEL_2c2bee16f94247f69bbbb15917e97d86"
}
},
"c3f50786ba7c45f1b586a3b854b1aaac": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_598ca6fa3c9a4ad2a3f17982a7034fde",
"placeholder": "​",
"style": "IPY_MODEL_360a24e2135d4fa2b5ed9c2a63621b43",
"value": "tokenizer_config.json: "
}
},
"70a4a5ad30b043b6b2d0b0761843dd37": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_a5041b01e6e145919265cc6bda5389a5",
"max": 1,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_e21801a4251d41aab4de18643b631e0c",
"value": 1
}
},
"e9b5b37bf4064247bf5eff86e294284a": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_30514c8dcff94761ad06b555e7bf9b90",
"placeholder": "​",
"style": "IPY_MODEL_bced9f1326aa4cd897992f723d547c85",
"value": " 7.30k/? [00:00<00:00, 402kB/s]"
}
},
"2c2bee16f94247f69bbbb15917e97d86": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"598ca6fa3c9a4ad2a3f17982a7034fde": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"360a24e2135d4fa2b5ed9c2a63621b43": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"a5041b01e6e145919265cc6bda5389a5": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": "20px"
}
},
"e21801a4251d41aab4de18643b631e0c": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"30514c8dcff94761ad06b555e7bf9b90": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"bced9f1326aa4cd897992f723d547c85": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"f092287399fc4c07951e6cc4d572fbcf": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_1c964adc152a4fecb480cf7d36172977",
"IPY_MODEL_a1ff644a085445d8869fa39cf582763d",
"IPY_MODEL_b96dd616fe334e069537d14008108891"
],
"layout": "IPY_MODEL_f800c31b4b8c448680dadc64048f64d1"
}
},
"1c964adc152a4fecb480cf7d36172977": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_d8af1760a74f4a8595d69dad31f7c955",
"placeholder": "​",
"style": "IPY_MODEL_8a02f56c23ed495f88e5f248c72a852e",
"value": "vocab.json: "
}
},
"a1ff644a085445d8869fa39cf582763d": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_e2f0b854b07043238247a7ed352a30b2",
"max": 1,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_500009082564414db7e821a7f5229a64",
"value": 1
}
},
"b96dd616fe334e069537d14008108891": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_d4d5b40c1e0d42e3af2a84db673150c3",
"placeholder": "​",
"style": "IPY_MODEL_91e15523847346629b2e6d249333f865",
"value": " 2.78M/? [00:00<00:00, 47.1MB/s]"
}
},
"f800c31b4b8c448680dadc64048f64d1": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"d8af1760a74f4a8595d69dad31f7c955": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"8a02f56c23ed495f88e5f248c72a852e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"e2f0b854b07043238247a7ed352a30b2": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": "20px"
}
},
"500009082564414db7e821a7f5229a64": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"d4d5b40c1e0d42e3af2a84db673150c3": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"91e15523847346629b2e6d249333f865": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"bb499be3f9174817adc4dd948b2d86ad": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_bbaa012346d149ecab17a63bf4fc23ee",
"IPY_MODEL_22a6f4cafa2a47a5947ac2add1f93670",
"IPY_MODEL_018dac22d66347bcafdad9cd63102c5e"
],
"layout": "IPY_MODEL_ea5780f2d85d428881a424c7613e8c91"
}
},
"bbaa012346d149ecab17a63bf4fc23ee": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_3db81fb226434f1c8e6cfd828f1c4c49",
"placeholder": "​",
"style": "IPY_MODEL_cb8e2f62c43a4acf8d8c9355f47d224c",
"value": "merges.txt: "
}
},
"22a6f4cafa2a47a5947ac2add1f93670": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_e6956a9bc99544809f1b50563f7cdda0",
"max": 1,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_443f1881bc7e469f96c195456b10ea4f",
"value": 1
}
},
"018dac22d66347bcafdad9cd63102c5e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_59651320ff594f34aed45b5a634ec80c",
"placeholder": "​",
"style": "IPY_MODEL_9111196283684fc195b3ed2efcdd30fa",
"value": " 1.67M/? [00:00<00:00, 34.4MB/s]"
}
},
"ea5780f2d85d428881a424c7613e8c91": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"3db81fb226434f1c8e6cfd828f1c4c49": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"cb8e2f62c43a4acf8d8c9355f47d224c": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"e6956a9bc99544809f1b50563f7cdda0": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": "20px"
}
},
"443f1881bc7e469f96c195456b10ea4f": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"59651320ff594f34aed45b5a634ec80c": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"9111196283684fc195b3ed2efcdd30fa": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"a92df6bd362f4751b1d95dd62e297063": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_72e803bfb66c47fcabceab5cb0a7709c",
"IPY_MODEL_22faefb7c130491e92572aa7989988b3",
"IPY_MODEL_6d38fbc61fe24910b6d5f952f5a0849c"
],
"layout": "IPY_MODEL_58e29408f3bd498d93687a180933d38d"
}
},
"72e803bfb66c47fcabceab5cb0a7709c": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_8ab673e23bfa4939a882219ab03713d3",
"placeholder": "​",
"style": "IPY_MODEL_00d5269a61dd47d4b3a331195d8d067e",
"value": "tokenizer.json: "
}
},
"22faefb7c130491e92572aa7989988b3": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_26d8bb1ca14c46738184175c1273f285",
"max": 1,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_c103614c36cc44eca9217bf343453cfd",
"value": 1
}
},
"6d38fbc61fe24910b6d5f952f5a0849c": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_ac5b6dd30e30451faebb7456754c51c8",
"placeholder": "​",
"style": "IPY_MODEL_a3a1316794b44adcbf1aada36d21fd42",
"value": " 7.03M/? [00:00<00:00, 67.5MB/s]"
}
},
"58e29408f3bd498d93687a180933d38d": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"8ab673e23bfa4939a882219ab03713d3": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"00d5269a61dd47d4b3a331195d8d067e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"26d8bb1ca14c46738184175c1273f285": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": "20px"
}
},
"c103614c36cc44eca9217bf343453cfd": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"ac5b6dd30e30451faebb7456754c51c8": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"a3a1316794b44adcbf1aada36d21fd42": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"ff893a34731c44aaabb159930cfd4194": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_e7ff8ab409384a55b219b6d6b14433c9",
"IPY_MODEL_453d4f43df2c4df5811b694a86005a39",
"IPY_MODEL_0eac9d1b9ae24295b51d3309427916a3"
],
"layout": "IPY_MODEL_3565272bc98c4cc2a16b822293bf6b59"
}
},
"e7ff8ab409384a55b219b6d6b14433c9": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_e611143c7ed944e0aedd23ad63d4e8b3",
"placeholder": "​",
"style": "IPY_MODEL_65863e96902942ddaa419d01d49123ce",
"value": "config.json: 100%"
}
},
"453d4f43df2c4df5811b694a86005a39": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_eca4c41da4b7442dadf8fc7a2a0bf388",
"max": 659,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_bf7b8b1ebe5542f0b776e4f6b6bef635",
"value": 659
}
},
"0eac9d1b9ae24295b51d3309427916a3": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_464b5836553a410c938e850eea077890",
"placeholder": "​",
"style": "IPY_MODEL_fcd103298f004a42a4ba6d9e902e8004",
"value": " 659/659 [00:00<00:00, 52.6kB/s]"
}
},
"3565272bc98c4cc2a16b822293bf6b59": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"e611143c7ed944e0aedd23ad63d4e8b3": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"65863e96902942ddaa419d01d49123ce": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"eca4c41da4b7442dadf8fc7a2a0bf388": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"bf7b8b1ebe5542f0b776e4f6b6bef635": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"464b5836553a410c938e850eea077890": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"fcd103298f004a42a4ba6d9e902e8004": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"a80f18c20a574f51b00bde6d960c27a9": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_5456e0c445914711b5aedee89c9bd499",
"IPY_MODEL_ea7168412eca44a4b1eb5d6a51b66d80",
"IPY_MODEL_130ee0490dd448b7aee41d973b1a1082"
],
"layout": "IPY_MODEL_75fbc8bbada049ab9f5dbfc73727422e"
}
},
"5456e0c445914711b5aedee89c9bd499": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_04cabcd6a2374793a2fc4f9d0186aa48",
"placeholder": "​",
"style": "IPY_MODEL_0a2fd3dae84443bcb3e6d7f739e598d5",
"value": "model.safetensors: 100%"
}
},
"ea7168412eca44a4b1eb5d6a51b66d80": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_7fbc129786fa4ed6b2cc5e4c60c018d0",
"max": 988097824,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_31d7a9b6b1a24fbea2f70a8b1e8bb7b5",
"value": 988097824
}
},
"130ee0490dd448b7aee41d973b1a1082": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_63cf1c6c9d30493ab0c721a196e0ee1e",
"placeholder": "​",
"style": "IPY_MODEL_ec69dc7bdc8d4bf0aa1a4fde84f89673",
"value": " 988M/988M [00:15<00:00, 85.4MB/s]"
}
},
"75fbc8bbada049ab9f5dbfc73727422e": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"04cabcd6a2374793a2fc4f9d0186aa48": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"0a2fd3dae84443bcb3e6d7f739e598d5": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"7fbc129786fa4ed6b2cc5e4c60c018d0": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"31d7a9b6b1a24fbea2f70a8b1e8bb7b5": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"63cf1c6c9d30493ab0c721a196e0ee1e": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"ec69dc7bdc8d4bf0aa1a4fde84f89673": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"00a9746486cf4c95a047fbbd0bc06ba4": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_2eb5d74bdcf94efaa954a77ceade4c4e",
"IPY_MODEL_34af92e1744d4e988bf2d52a6e8a47a5",
"IPY_MODEL_9fe722aa69a04e20a5c3a054694005a1"
],
"layout": "IPY_MODEL_3c73815fa69441309f19b82ba961b0e3"
}
},
"2eb5d74bdcf94efaa954a77ceade4c4e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_950c9855ef344481a4011f462255788c",
"placeholder": "​",
"style": "IPY_MODEL_365579db13964dccadb117805a032b37",
"value": "generation_config.json: 100%"
}
},
"34af92e1744d4e988bf2d52a6e8a47a5": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_eb3d00bd47cd4349a84e11396db4c3e8",
"max": 242,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_c1a35560bf3a4c2292db5f0c7ad78987",
"value": 242
}
},
"9fe722aa69a04e20a5c3a054694005a1": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_400304ad2e5f40519e371cdc342146e5",
"placeholder": "​",
"style": "IPY_MODEL_5e23bdf5b19e4d78b9baaa6d25661f6d",
"value": " 242/242 [00:00<00:00, 11.2kB/s]"
}
},
"3c73815fa69441309f19b82ba961b0e3": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"950c9855ef344481a4011f462255788c": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"365579db13964dccadb117805a032b37": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"eb3d00bd47cd4349a84e11396db4c3e8": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"c1a35560bf3a4c2292db5f0c7ad78987": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"400304ad2e5f40519e371cdc342146e5": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"5e23bdf5b19e4d78b9baaa6d25661f6d": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
}
}
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/alonsosilvaallende/f1daa7b0debd26ea6830b48cef3b1282/understanding_next_token_prediction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"source": [
"## Understanding Next Token Prediction"
],
"metadata": {
"id": "v18e2XKwRKN2"
}
},
{
"cell_type": "markdown",
"source": [
"Let's look at a [next token prediction app](https://huggingface.co/spaces/alonsosilva/NextTokenPrediction).\n",
"\n",
"This notebooks is mostly taken from [How Transformer LLMs work](https://learn.deeplearning.ai/courses/how-transformer-llms-work)."
],
"metadata": {
"id": "jlf4We0xQ2-n"
}
},
{
"cell_type": "markdown",
"source": [
"Let's first load the model and its tokenizer. For that you will first import the classes: AutoModelForCausalLM and AutoTokenizer. When you want to process a sentence, you can apply the tokenizer first and then the model in two separate steps. Or you can create a pipeline object that wraps the two steps and then apply the pipeline to the sentence. You'll explore both approaches in this notebook. This is why you'll also import the pipeline class."
],
"metadata": {
"id": "GLhdsVeRMY3V"
}
},
{
"cell_type": "code",
"source": [
"import torch\n",
"from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline\n",
"\n",
"MODEL_ID = \"Qwen/Qwen2.5-0.5B-Instruct\"\n",
"tokenizer = AutoTokenizer.from_pretrained(\n",
" MODEL_ID,\n",
" cache_dir=\"/content/drive/My Drive/\",\n",
")\n",
"model = AutoModelForCausalLM.from_pretrained(\n",
" MODEL_ID,\n",
" cache_dir=\"/content/drive/My Drive/\",\n",
" device_map=\"cpu\",\n",
" torch_dtype=\"auto\",\n",
" trust_remote_code=True,\n",
")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 388,
"referenced_widgets": [
"3b8586dace7a4005a07ee8c67f80c1f4",
"c3f50786ba7c45f1b586a3b854b1aaac",
"70a4a5ad30b043b6b2d0b0761843dd37",
"e9b5b37bf4064247bf5eff86e294284a",
"2c2bee16f94247f69bbbb15917e97d86",
"598ca6fa3c9a4ad2a3f17982a7034fde",
"360a24e2135d4fa2b5ed9c2a63621b43",
"a5041b01e6e145919265cc6bda5389a5",
"e21801a4251d41aab4de18643b631e0c",
"30514c8dcff94761ad06b555e7bf9b90",
"bced9f1326aa4cd897992f723d547c85",
"f092287399fc4c07951e6cc4d572fbcf",
"1c964adc152a4fecb480cf7d36172977",
"a1ff644a085445d8869fa39cf582763d",
"b96dd616fe334e069537d14008108891",
"f800c31b4b8c448680dadc64048f64d1",
"d8af1760a74f4a8595d69dad31f7c955",
"8a02f56c23ed495f88e5f248c72a852e",
"e2f0b854b07043238247a7ed352a30b2",
"500009082564414db7e821a7f5229a64",
"d4d5b40c1e0d42e3af2a84db673150c3",
"91e15523847346629b2e6d249333f865",
"bb499be3f9174817adc4dd948b2d86ad",
"bbaa012346d149ecab17a63bf4fc23ee",
"22a6f4cafa2a47a5947ac2add1f93670",
"018dac22d66347bcafdad9cd63102c5e",
"ea5780f2d85d428881a424c7613e8c91",
"3db81fb226434f1c8e6cfd828f1c4c49",
"cb8e2f62c43a4acf8d8c9355f47d224c",
"e6956a9bc99544809f1b50563f7cdda0",
"443f1881bc7e469f96c195456b10ea4f",
"59651320ff594f34aed45b5a634ec80c",
"9111196283684fc195b3ed2efcdd30fa",
"a92df6bd362f4751b1d95dd62e297063",
"72e803bfb66c47fcabceab5cb0a7709c",
"22faefb7c130491e92572aa7989988b3",
"6d38fbc61fe24910b6d5f952f5a0849c",
"58e29408f3bd498d93687a180933d38d",
"8ab673e23bfa4939a882219ab03713d3",
"00d5269a61dd47d4b3a331195d8d067e",
"26d8bb1ca14c46738184175c1273f285",
"c103614c36cc44eca9217bf343453cfd",
"ac5b6dd30e30451faebb7456754c51c8",
"a3a1316794b44adcbf1aada36d21fd42",
"ff893a34731c44aaabb159930cfd4194",
"e7ff8ab409384a55b219b6d6b14433c9",
"453d4f43df2c4df5811b694a86005a39",
"0eac9d1b9ae24295b51d3309427916a3",
"3565272bc98c4cc2a16b822293bf6b59",
"e611143c7ed944e0aedd23ad63d4e8b3",
"65863e96902942ddaa419d01d49123ce",
"eca4c41da4b7442dadf8fc7a2a0bf388",
"bf7b8b1ebe5542f0b776e4f6b6bef635",
"464b5836553a410c938e850eea077890",
"fcd103298f004a42a4ba6d9e902e8004",
"a80f18c20a574f51b00bde6d960c27a9",
"5456e0c445914711b5aedee89c9bd499",
"ea7168412eca44a4b1eb5d6a51b66d80",
"130ee0490dd448b7aee41d973b1a1082",
"75fbc8bbada049ab9f5dbfc73727422e",
"04cabcd6a2374793a2fc4f9d0186aa48",
"0a2fd3dae84443bcb3e6d7f739e598d5",
"7fbc129786fa4ed6b2cc5e4c60c018d0",
"31d7a9b6b1a24fbea2f70a8b1e8bb7b5",
"63cf1c6c9d30493ab0c721a196e0ee1e",
"ec69dc7bdc8d4bf0aa1a4fde84f89673",
"00a9746486cf4c95a047fbbd0bc06ba4",
"2eb5d74bdcf94efaa954a77ceade4c4e",
"34af92e1744d4e988bf2d52a6e8a47a5",
"9fe722aa69a04e20a5c3a054694005a1",
"3c73815fa69441309f19b82ba961b0e3",
"950c9855ef344481a4011f462255788c",
"365579db13964dccadb117805a032b37",
"eb3d00bd47cd4349a84e11396db4c3e8",
"c1a35560bf3a4c2292db5f0c7ad78987",
"400304ad2e5f40519e371cdc342146e5",
"5e23bdf5b19e4d78b9baaa6d25661f6d"
]
},
"id": "T4Pj4vfggsxE",
"outputId": "1093429c-43c4-4858-aa2a-c17c89066b01"
},
"execution_count": 1,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"WARNING:torchao.kernel.intmm:Warning: Detected no triton, on systems without Triton certain kernels will not work\n",
"/usr/local/lib/python3.12/dist-packages/huggingface_hub/utils/_auth.py:104: UserWarning: \n",
"Error while fetching `HF_TOKEN` secret value from your vault: 'Requesting secret HF_TOKEN timed out. Secrets can only be fetched when running from the Colab UI.'.\n",
"You are not authenticated with the Hugging Face Hub in this notebook.\n",
"If the error persists, please let us know by opening an issue on GitHub (https://github.com/huggingface/huggingface_hub/issues/new).\n",
" warnings.warn(\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"tokenizer_config.json: 0.00B [00:00, ?B/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "3b8586dace7a4005a07ee8c67f80c1f4"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"vocab.json: 0.00B [00:00, ?B/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "f092287399fc4c07951e6cc4d572fbcf"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"merges.txt: 0.00B [00:00, ?B/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "bb499be3f9174817adc4dd948b2d86ad"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"tokenizer.json: 0.00B [00:00, ?B/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "a92df6bd362f4751b1d95dd62e297063"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"config.json: 0%| | 0.00/659 [00:00<?, ?B/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "ff893a34731c44aaabb159930cfd4194"
}
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"`torch_dtype` is deprecated! Use `dtype` instead!\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"model.safetensors: 0%| | 0.00/988M [00:00<?, ?B/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "a80f18c20a574f51b00bde6d960c27a9"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"generation_config.json: 0%| | 0.00/242 [00:00<?, ?B/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "00a9746486cf4c95a047fbbd0bc06ba4"
}
},
"metadata": {}
}
]
},
{
"cell_type": "markdown",
"source": [
"Now you can wrap the model and the tokenizer in a [pipeline](https://huggingface.co/docs/transformers/en/main_classes/pipelines#transformers.pipeline) object that has \"text-generation\" as task."
],
"metadata": {
"id": "qQfsQz9BMjKs"
}
},
{
"cell_type": "code",
"source": [
"# Create a pipeline\n",
"generator = pipeline(\n",
" \"text-generation\",\n",
" model=model,\n",
" tokenizer=tokenizer,\n",
" return_full_text=False, # False means to not include the prompt text in the returned text\n",
" max_new_tokens=50,\n",
" do_sample=False, # no randomness in the generated text\n",
")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "kbQi3M_VgLbH",
"outputId": "65ecfb73-0dca-46f2-e062-ce26cdc5686d"
},
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Device set to use cpu\n",
"The following generation flags are not valid and may be ignored: ['temperature', 'top_p', 'top_k']. Set `TRANSFORMERS_VERBOSITY=info` for more details.\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"We use the pipeline object (labeled as generator) to generate a response consisting of 50 tokens to the given prompt."
],
"metadata": {
"id": "JU39CRqMNs_s"
}
},
{
"cell_type": "code",
"source": [
"prompt = \"The capital of France is\"\n",
"output = generator(prompt)\n",
"print(output[0]['generated_text'])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "-ei8TOywgNvS",
"outputId": "78608c4e-8b6c-40a8-cee1-e537080357d1"
},
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" Paris. It was founded in 789 AD by Charlemagne, the last king of the Carolingian dynasty. The city has a long and rich history dating back to ancient times. In fact, it's one of the oldest cities\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"messages = [{\"role\": \"user\", \"content\": \"Hi!\"}]\n",
"output = generator(messages)\n",
"print(output[0]['generated_text'])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "_HRdylyuhmoV",
"outputId": "6fcd1e7a-2411-4c8f-b373-bea191796e4e"
},
"execution_count": 5,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Hello! How can I assist you today?\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
" We can print the model to take a look at its architecture."
],
"metadata": {
"id": "nZic8ldAMtzi"
}
},
{
"cell_type": "code",
"source": [
"model"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "xCRsErxyiBgx",
"outputId": "afa85df8-fbca-4f1c-e045-9c2527f26391"
},
"execution_count": 3,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Qwen2ForCausalLM(\n",
" (model): Qwen2Model(\n",
" (embed_tokens): Embedding(151936, 896)\n",
" (layers): ModuleList(\n",
" (0-23): 24 x Qwen2DecoderLayer(\n",
" (self_attn): Qwen2Attention(\n",
" (q_proj): Linear(in_features=896, out_features=896, bias=True)\n",
" (k_proj): Linear(in_features=896, out_features=128, bias=True)\n",
" (v_proj): Linear(in_features=896, out_features=128, bias=True)\n",
" (o_proj): Linear(in_features=896, out_features=896, bias=False)\n",
" )\n",
" (mlp): Qwen2MLP(\n",
" (gate_proj): Linear(in_features=896, out_features=4864, bias=False)\n",
" (up_proj): Linear(in_features=896, out_features=4864, bias=False)\n",
" (down_proj): Linear(in_features=4864, out_features=896, bias=False)\n",
" (act_fn): SiLUActivation()\n",
" )\n",
" (input_layernorm): Qwen2RMSNorm((896,), eps=1e-06)\n",
" (post_attention_layernorm): Qwen2RMSNorm((896,), eps=1e-06)\n",
" )\n",
" )\n",
" (norm): Qwen2RMSNorm((896,), eps=1e-06)\n",
" (rotary_emb): Qwen2RotaryEmbedding()\n",
" )\n",
" (lm_head): Linear(in_features=896, out_features=151936, bias=False)\n",
")"
]
},
"metadata": {},
"execution_count": 3
}
]
},
{
"cell_type": "code",
"source": [
"model.model.embed_tokens"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "nODdrv-Wis_I",
"outputId": "b6b295ff-513c-4177-f50e-c0531e87c14f"
},
"execution_count": 4,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Embedding(151936, 896)"
]
},
"metadata": {},
"execution_count": 4
}
]
},
{
"cell_type": "markdown",
"source": [
"Are these tokens nearby in the embedding space?"
],
"metadata": {
"id": "WZQUXoi4b2wW"
}
},
{
"cell_type": "code",
"source": [
"words = [\" forest\", \"Hello\", \" hello\", \" Hello\", \"hello\"]"
],
"metadata": {
"id": "ki2kKHBNcWGc"
},
"execution_count": 9,
"outputs": []
},
{
"cell_type": "code",
"source": [
"token_ids = [tokenizer.encode(word)[0] for word in words]\n",
"token_ids"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "viQNtLoacsnh",
"outputId": "92ae86dc-a713-4b84-b79d-aae121d99129"
},
"execution_count": 18,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[13638, 9707, 23811, 21927, 14990]"
]
},
"metadata": {},
"execution_count": 18
}
]
},
{
"cell_type": "code",
"source": [
"# Get the output of the embed tokens layer\n",
"model_output = model.model.embed_tokens(torch.tensor([token_ids]))"
],
"metadata": {
"id": "VMyiTfTHjNy3"
},
"execution_count": 19,
"outputs": []
},
{
"cell_type": "code",
"source": [
"def my_distance(a, b):\n",
" return ((a-b)**2).sum(axis=0).item()"
],
"metadata": {
"id": "WhPL-41zkvAA"
},
"execution_count": 20,
"outputs": []
},
{
"cell_type": "code",
"source": [
"for i in range(len(model_output[0])):\n",
" for j in range(i+1, len(model_output[0])):\n",
" print(f\"Distance between {words[i]} and {words[j]}: {my_distance(model_output[0][i], model_output[0][j])}\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "496PWKYXk90m",
"outputId": "c5457382-a1c9-44c7-b128-05ec41598f03"
},
"execution_count": 21,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Distance between forest and Hello: 0.3125\n",
"Distance between forest and hello: 0.30859375\n",
"Distance between forest and Hello: 0.302734375\n",
"Distance between forest and hello: 0.322265625\n",
"Distance between Hello and hello: 0.123046875\n",
"Distance between Hello and Hello: 0.0712890625\n",
"Distance between Hello and hello: 0.11474609375\n",
"Distance between hello and Hello: 0.10302734375\n",
"Distance between hello and hello: 0.0849609375\n",
"Distance between Hello and hello: 0.126953125\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"Let's give the model a prompt and check the first token it will generate."
],
"metadata": {
"id": "QUgodUQMJ4bf"
}
},
{
"cell_type": "code",
"source": [
"prompt = \"The capital of France is\""
],
"metadata": {
"id": "lRbjkjIgjBmI"
},
"execution_count": 7,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"We first tokenize the prompt and get the ids of the tokens."
],
"metadata": {
"id": "w4cnE63HKFby"
}
},
{
"cell_type": "code",
"source": [
"input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
"input_ids"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Bl_G0vY1jNCM",
"outputId": "1a89373d-b592-4392-b195-17e2c3f9c542"
},
"execution_count": 8,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"tensor([[ 785, 6722, 315, 9625, 374]])"
]
},
"metadata": {},
"execution_count": 8
}
]
},
{
"cell_type": "markdown",
"source": [
"We pass the token ids to the transformer block (before the LM head)."
],
"metadata": {
"id": "_2niwu6WKgSe"
}
},
{
"cell_type": "code",
"source": [
"# Get the output of the model before the lm_head\n",
"model_output = model.model(input_ids)"
],
"metadata": {
"id": "j8s7in1FjcnX"
},
"execution_count": 9,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"The transformer block outputs for each token a vector of size 896 (embedding size). Let's check the shape of this output."
],
"metadata": {
"id": "zwCeboV-K8n4"
}
},
{
"cell_type": "code",
"source": [
"# Get the shape the output the model before the lm_head\n",
"model_output[0].shape"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "U1XwiPxKluSK",
"outputId": "e24f9ca3-d193-4a4a-bc0f-6b6d2b1b03c9"
},
"execution_count": 10,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"torch.Size([1, 5, 896])"
]
},
"metadata": {},
"execution_count": 10
}
]
},
{
"cell_type": "markdown",
"source": [
"The first number represents the batch size, which is 1 in this case since we have one prompt. The second number 5 represents the number of tokens. And finally 896 represents the embedding size (the size of the vector that corresponds to each token).\n",
"\n",
"Let's now get the output of the LM head."
],
"metadata": {
"id": "UqvhVK80LbVJ"
}
},
{
"cell_type": "code",
"source": [
"# Get the output of the lm_head\n",
"lm_head_output = model.lm_head(model_output[0])"
],
"metadata": {
"id": "nCfW2ooelz7H"
},
"execution_count": 11,
"outputs": []
},
{
"cell_type": "code",
"source": [
"lm_head_output.shape"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "RWpH1KfeOogz",
"outputId": "f9941025-d7c6-4824-9cf5-9498497d7252"
},
"execution_count": 12,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"torch.Size([1, 5, 151936])"
]
},
"metadata": {},
"execution_count": 12
}
]
},
{
"cell_type": "markdown",
"source": [
"[https://x.com/gabriberton/status/2007268853072720266](https://x.com/gabriberton/status/2007268853072720266)\n",
"\n",
"[https://x.com/gabriberton/status/2007327212438204893?s=20](https://x.com/gabriberton/status/2007327212438204893?s=20)"
],
"metadata": {
"id": "UbpSDRfyP4ec"
}
},
{
"cell_type": "markdown",
"source": [
"The LM head outputs for each token in the input prompt, a vector of size 151936 (padded vocabulary size). So there are 5 vectors, each of size 151936. Each vector can be mapped to a probability distribution, that shows the probability for each token in the vocabulary to come after the given token in the input prompt.\n",
"\n",
"Since we're interested in generating the output token that comes after the last token in the input prompt (\"is\"), we'll focus on the last vector. So in the next cell, lm_head_output[0,-1] is a vector of size 151936 from which you can generate the token that comes after (\"is\"). You can do that by finding the id of the token that corresponds to the highest value in the vector lm_head_output[0,-1] (using argmax(-1), -1 means across the last axis here)."
],
"metadata": {
"id": "1dd8N9CHL7QX"
}
},
{
"cell_type": "code",
"source": [
"token_id = lm_head_output[0,-1].argmax(-1)\n",
"token_id"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "t5b3SOj3l86w",
"outputId": "0d728e72-1b37-44c3-fba7-30a3d6089cf9"
},
"execution_count": 27,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"tensor(12095)"
]
},
"metadata": {},
"execution_count": 27
}
]
},
{
"cell_type": "markdown",
"source": [
"Finally, let's decode the returned token id."
],
"metadata": {
"id": "DBRzC8dZLxjb"
}
},
{
"cell_type": "code",
"source": [
"tokenizer.decode(token_id)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 36
},
"id": "k5wSp8wVmE1O",
"outputId": "779acee5-05ec-4f45-f197-904a645451a1"
},
"execution_count": 28,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"' Paris'"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "string"
}
},
"metadata": {},
"execution_count": 28
}
]
},
{
"cell_type": "code",
"source": [
"prompt = \"The sky is blue because\""
],
"metadata": {
"id": "a67p7avwd6jW"
},
"execution_count": 29,
"outputs": []
},
{
"cell_type": "code",
"source": [
"input_ids = tokenizer(prompt, return_tensors=\"pt\").input_ids\n",
"input_ids"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "MLxb3RWrd46C",
"outputId": "b4dad68b-8118-4c3a-c325-4405f7f5d359"
},
"execution_count": 30,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"tensor([[ 785, 12884, 374, 6303, 1576]])"
]
},
"metadata": {},
"execution_count": 30
}
]
},
{
"cell_type": "code",
"source": [
"def set_seed(seed: int = 42):\n",
" # random.seed(seed)\n",
" # np.random.seed(seed)\n",
" torch.manual_seed(seed)\n",
" torch.cuda.manual_seed(seed)\n",
" # torch.cuda.manual_seed_all(seed) # for multi-GPU"
],
"metadata": {
"id": "g0v3zknTotyV"
},
"execution_count": 33,
"outputs": []
},
{
"cell_type": "code",
"source": [
"full_response = input_ids\n",
"set_seed(40)\n",
"for generated_token in range(20):\n",
" with torch.no_grad():\n",
" outputs = model(full_response)\n",
" logits = outputs.logits[:, -1, :] # Focus on the last token's logits\n",
" probs = torch.softmax(logits, dim=-1)\n",
" sampled_token = torch.multinomial(probs, num_samples=1)\n",
" print(tokenizer.decode(sampled_token[0]), end=\"\")\n",
" full_response = torch.cat([full_response, sampled_token], dim=-1)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "PBAvDG-VeHH0",
"outputId": "7450a0d8-3a55-4cff-c105-43775b028199"
},
"execution_count": 38,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" the Earth reflects sunlight. It made the trees look bright green and blue. This makes the blue sky"
]
}
]
},
{
"cell_type": "code",
"source": [
"outputs = model(input_ids)\n",
"logits = outputs.logits[:, -1, :] # Focus on the last token's logits"
],
"metadata": {
"id": "6KLn8l53mJi2"
},
"execution_count": 50,
"outputs": []
},
{
"cell_type": "code",
"source": [
"logits"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "kKmUmGLemk-I",
"outputId": "1bea6452-22df-4f18-aa8b-8486cc2cdf91"
},
"execution_count": 51,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"tensor([[ 5.1562, 8.1875, 3.2344, ..., -3.2969, -3.2969, -3.2969]],\n",
" dtype=torch.bfloat16, grad_fn=<SelectBackward0>)"
]
},
"metadata": {},
"execution_count": 51
}
]
},
{
"cell_type": "code",
"source": [
"len(logits[0])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "mlP--AnymmBa",
"outputId": "ca097f7a-ec88-4269-e68c-6945926aa046"
},
"execution_count": 52,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"151936"
]
},
"metadata": {},
"execution_count": 52
}
]
},
{
"cell_type": "code",
"source": [
"len(tokenizer)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "lcObE9gemyRO",
"outputId": "b49d5a8e-d024-485c-a4a1-fa06717ea27b"
},
"execution_count": 53,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"151669"
]
},
"metadata": {},
"execution_count": 53
}
]
},
{
"cell_type": "code",
"source": [
"token_ids = [token_id for token_id in range(len(tokenizer))]\n"
],
"metadata": {
"id": "UZTfYXblmz1c"
},
"execution_count": 54,
"outputs": []
},
{
"cell_type": "code",
"source": [
"tokens = [tokenizer.decode(token_id) for token_id in token_ids]"
],
"metadata": {
"id": "Tuu5A_93nJ7h"
},
"execution_count": 55,
"outputs": []
},
{
"cell_type": "code",
"source": [
"logits_list = logits[0][:len(tokenizer)].tolist()"
],
"metadata": {
"id": "x_TEbmKpnMS1"
},
"execution_count": 61,
"outputs": []
},
{
"cell_type": "code",
"source": [
"logits_list[:10]"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "WbvIYq3QnOOt",
"outputId": "b73d47fc-275d-4241-b164-d34203f11ce4"
},
"execution_count": 62,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"[5.15625,\n",
" 8.1875,\n",
" 3.234375,\n",
" 4.96875,\n",
" 6.78125,\n",
" 1.890625,\n",
" 6.84375,\n",
" 7.40625,\n",
" 4.40625,\n",
" 6.375]"
]
},
"metadata": {},
"execution_count": 62
}
]
},
{
"cell_type": "code",
"source": [
"probs = torch.softmax(logits, dim=-1)"
],
"metadata": {
"id": "17fLqd47nbJF"
},
"execution_count": 64,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import pandas as pd"
],
"metadata": {
"id": "UR9SQ8efnuEH"
},
"execution_count": 65,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df = pd.DataFrame({\"token_id\": token_ids, \"token\": tokens, \"probability\": probs[0][:len(tokenizer)].tolist()})"
],
"metadata": {
"id": "KP4FCIVHnzHu"
},
"execution_count": 67,
"outputs": []
},
{
"cell_type": "code",
"source": [
"df.head()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 206
},
"id": "MAaeiaEPn4Y3",
"outputId": "42c5d557-1261-462d-dbfa-0605ac3e8b7b"
},
"execution_count": 68,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" token_id token probability\n",
"0 0 ! 3.099442e-06\n",
"1 1 \" 6.437302e-05\n",
"2 2 # 4.544854e-07\n",
"3 3 $ 2.577901e-06\n",
"4 4 % 1.573563e-05"
],
"text/html": [
"\n",
" <div id=\"df-23668144-f1dc-4f98-96d8-5683186c9bb7\" class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>token_id</th>\n",
" <th>token</th>\n",
" <th>probability</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0</td>\n",
" <td>!</td>\n",
" <td>3.099442e-06</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>\"</td>\n",
" <td>6.437302e-05</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>2</td>\n",
" <td>#</td>\n",
" <td>4.544854e-07</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3</td>\n",
" <td>$</td>\n",
" <td>2.577901e-06</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>4</td>\n",
" <td>%</td>\n",
" <td>1.573563e-05</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <div class=\"colab-df-buttons\">\n",
"\n",
" <div class=\"colab-df-container\">\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-23668144-f1dc-4f98-96d8-5683186c9bb7')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" .colab-df-buttons div {\n",
" margin-bottom: 4px;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-23668144-f1dc-4f98-96d8-5683186c9bb7 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-23668144-f1dc-4f98-96d8-5683186c9bb7');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
"\n",
"\n",
" </div>\n",
" </div>\n"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "dataframe",
"variable_name": "df"
}
},
"metadata": {},
"execution_count": 68
}
]
},
{
"cell_type": "code",
"source": [
"df.sort_values(by=\"probability\", ascending=False).head(10)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 362
},
"id": "9hYzNjOwn8s6",
"outputId": "cfc580f6-3885-46ea-8b1b-b35a9fe300f1"
},
"execution_count": 69,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" token_id token probability\n",
"12095 12095 Paris 0.628906\n",
"7407 7407 located 0.031250\n",
"279 279 the 0.022949\n",
"1112 1112 ... 0.016724\n",
"30743 30743 ____ 0.015747\n",
"32671 32671 ______ 0.014771\n",
"220 220 0.013916\n",
"304 304 in 0.012268\n",
"1304 1304 __ 0.009583\n",
"2130 2130 ____ 0.009583"
],
"text/html": [
"\n",
" <div id=\"df-91b423bd-6c87-427c-8e8f-a42cdc2eb09e\" class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>token_id</th>\n",
" <th>token</th>\n",
" <th>probability</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>12095</th>\n",
" <td>12095</td>\n",
" <td>Paris</td>\n",
" <td>0.628906</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7407</th>\n",
" <td>7407</td>\n",
" <td>located</td>\n",
" <td>0.031250</td>\n",
" </tr>\n",
" <tr>\n",
" <th>279</th>\n",
" <td>279</td>\n",
" <td>the</td>\n",
" <td>0.022949</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1112</th>\n",
" <td>1112</td>\n",
" <td>...</td>\n",
" <td>0.016724</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30743</th>\n",
" <td>30743</td>\n",
" <td>____</td>\n",
" <td>0.015747</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32671</th>\n",
" <td>32671</td>\n",
" <td>______</td>\n",
" <td>0.014771</td>\n",
" </tr>\n",
" <tr>\n",
" <th>220</th>\n",
" <td>220</td>\n",
" <td></td>\n",
" <td>0.013916</td>\n",
" </tr>\n",
" <tr>\n",
" <th>304</th>\n",
" <td>304</td>\n",
" <td>in</td>\n",
" <td>0.012268</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1304</th>\n",
" <td>1304</td>\n",
" <td>__</td>\n",
" <td>0.009583</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2130</th>\n",
" <td>2130</td>\n",
" <td>____</td>\n",
" <td>0.009583</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <div class=\"colab-df-buttons\">\n",
"\n",
" <div class=\"colab-df-container\">\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-91b423bd-6c87-427c-8e8f-a42cdc2eb09e')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" .colab-df-buttons div {\n",
" margin-bottom: 4px;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-91b423bd-6c87-427c-8e8f-a42cdc2eb09e button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-91b423bd-6c87-427c-8e8f-a42cdc2eb09e');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
"\n",
"\n",
" </div>\n",
" </div>\n"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "dataframe",
"summary": "{\n \"name\": \"df\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"token_id\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 12659,\n \"min\": 220,\n \"max\": 32671,\n \"num_unique_values\": 10,\n \"samples\": [\n 1304,\n 7407,\n 32671\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"token\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 10,\n \"samples\": [\n \" __\",\n \" located\",\n \" ______\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"probability\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.19382968790484356,\n \"min\": 0.00958251953125,\n \"max\": 0.62890625,\n \"num_unique_values\": 9,\n \"samples\": [\n 0.01226806640625,\n 0.03125,\n 0.0147705078125\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
}
},
"metadata": {},
"execution_count": 69
}
]
},
{
"cell_type": "code",
"source": [
"df[df['token'] == \" Brussels\"]"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 80
},
"id": "G9LBF5BTqO8t",
"outputId": "b1567463-7955-4c20-9332-bcb321d3b532"
},
"execution_count": 78,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
" token_id token probability\n",
"37169 37169 Brussels 0.002136"
],
"text/html": [
"\n",
" <div id=\"df-17375d0d-9243-456e-b08f-3aa5175cfa20\" class=\"colab-df-container\">\n",
" <div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>token_id</th>\n",
" <th>token</th>\n",
" <th>probability</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>37169</th>\n",
" <td>37169</td>\n",
" <td>Brussels</td>\n",
" <td>0.002136</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>\n",
" <div class=\"colab-df-buttons\">\n",
"\n",
" <div class=\"colab-df-container\">\n",
" <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-17375d0d-9243-456e-b08f-3aa5175cfa20')\"\n",
" title=\"Convert this dataframe to an interactive table.\"\n",
" style=\"display:none;\">\n",
"\n",
" <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
" <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
" </svg>\n",
" </button>\n",
"\n",
" <style>\n",
" .colab-df-container {\n",
" display:flex;\n",
" gap: 12px;\n",
" }\n",
"\n",
" .colab-df-convert {\n",
" background-color: #E8F0FE;\n",
" border: none;\n",
" border-radius: 50%;\n",
" cursor: pointer;\n",
" display: none;\n",
" fill: #1967D2;\n",
" height: 32px;\n",
" padding: 0 0 0 0;\n",
" width: 32px;\n",
" }\n",
"\n",
" .colab-df-convert:hover {\n",
" background-color: #E2EBFA;\n",
" box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
" fill: #174EA6;\n",
" }\n",
"\n",
" .colab-df-buttons div {\n",
" margin-bottom: 4px;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert {\n",
" background-color: #3B4455;\n",
" fill: #D2E3FC;\n",
" }\n",
"\n",
" [theme=dark] .colab-df-convert:hover {\n",
" background-color: #434B5C;\n",
" box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
" filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
" fill: #FFFFFF;\n",
" }\n",
" </style>\n",
"\n",
" <script>\n",
" const buttonEl =\n",
" document.querySelector('#df-17375d0d-9243-456e-b08f-3aa5175cfa20 button.colab-df-convert');\n",
" buttonEl.style.display =\n",
" google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
"\n",
" async function convertToInteractive(key) {\n",
" const element = document.querySelector('#df-17375d0d-9243-456e-b08f-3aa5175cfa20');\n",
" const dataTable =\n",
" await google.colab.kernel.invokeFunction('convertToInteractive',\n",
" [key], {});\n",
" if (!dataTable) return;\n",
"\n",
" const docLinkHtml = 'Like what you see? Visit the ' +\n",
" '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
" + ' to learn more about interactive tables.';\n",
" element.innerHTML = '';\n",
" dataTable['output_type'] = 'display_data';\n",
" await google.colab.output.renderOutput(dataTable, element);\n",
" const docLink = document.createElement('div');\n",
" docLink.innerHTML = docLinkHtml;\n",
" element.appendChild(docLink);\n",
" }\n",
" </script>\n",
" </div>\n",
"\n",
"\n",
" </div>\n",
" </div>\n"
],
"application/vnd.google.colaboratory.intrinsic+json": {
"type": "dataframe",
"summary": "{\n \"name\": \"df[df['token'] == \\\" Brussels\\\"]\",\n \"rows\": 1,\n \"fields\": [\n {\n \"column\": \"token_id\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 37169,\n \"max\": 37169,\n \"num_unique_values\": 1,\n \"samples\": [\n 37169\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"token\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 1,\n \"samples\": [\n \" Brussels\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"probability\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": null,\n \"min\": 0.00213623046875,\n \"max\": 0.00213623046875,\n \"num_unique_values\": 1,\n \"samples\": [\n 0.00213623046875\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
}
},
"metadata": {},
"execution_count": 78
}
]
},
{
"cell_type": "code",
"source": [
"set_seed(42)\n",
"sampled_token = torch.multinomial(probs, num_samples=1)\n",
"sampled_token"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "7BWhEN2-oHVW",
"outputId": "dc6b38ec-a7c0-4601-a654-8fb2d13d592b"
},
"execution_count": 73,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"tensor([[304]])"
]
},
"metadata": {},
"execution_count": 73
}
]
},
{
"cell_type": "code",
"source": [
"for seed in range(100):\n",
" set_seed(seed)\n",
" sampled_token = torch.multinomial(probs, num_samples=1)\n",
" print(f\"{seed}: {tokenizer.decode(sampled_token[0])}\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "onyYX3SUoy6b",
"outputId": "248db619-b60b-4253-f71c-ef6f7bf1eef2"
},
"execution_count": 77,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"0: ______\n",
"1: :\n",
"\n",
"2: Paris\n",
"3: located\n",
"4: Paris\n",
"5: Paris\n",
"6: ...\n",
"7: Paris\n",
"8: Paris\n",
"9: located\n",
"10: ____\n",
"11: Paris\n",
"12: Paris\n",
"13: ?\n",
"\n",
"\n",
"14: \n",
"15: Paris\n",
"16: Paris\n",
"17: ...\n",
"18: ...\n",
"19: Paris\n",
"20: prec\n",
"21: Paris\n",
"22: Paris\n",
"23: Paris\n",
"24: Paris\n",
"25: ____\n",
"26: Paris\n",
"27: Paris\n",
"28: capital\n",
"29: Paris\n",
"30: Paris\n",
"31: Paris\n",
"32: called\n",
"33: Paris\n",
"34: Paris\n",
"35: Paris\n",
"36: (\n",
"37: Paris\n",
"38: Paris\n",
"39: Paris\n",
"40: Paris\n",
"41: Paris\n",
"42: in\n",
"43: Paris\n",
"44: Paris\n",
"45: Paris\n",
"46: Paris\n",
"47: Paris\n",
"48: Rome\n",
"49: Brussels\n",
"50: Paris\n",
"51: Paris\n",
"52: Paris\n",
"53: located\n",
"54: Paris\n",
"55: Paris\n",
"56: ?\n",
"\n",
"57: :\n",
"\n",
"\n",
"58: Paris\n",
"59: ______\n",
"60: Paris\n",
"61: a\n",
"62: Paris\n",
"63: Paris\n",
"64: also\n",
"65: ____\n",
"66: Paris\n",
"67: Paris\n",
"68: Paris\n",
"69: Paris\n",
"70: ____\n",
"71: located\n",
"72: Paris\n",
"73: Paris\n",
"74: \n",
"75: Paris\n",
"76: Paris\n",
"77: Paris\n",
"78: Paris\n",
"79: Paris\n",
"80: commonly\n",
"81: located\n",
"82: Paris\n",
"83: Paris\n",
"84: Paris\n",
"85: Paris\n",
"86: Paris\n",
"87: Paris\n",
"88: Paris\n",
"89: Paris\n",
"90: (\n",
"91: Paris\n",
"92: France\n",
"93: the\n",
"94: Paris\n",
"95: Paris\n",
"96: Paris\n",
"97: Paris\n",
"98: .\n",
"99: Paris\n"
]
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "y2kEskUgpAv1"
},
"execution_count": null,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment