spontanious openwebui docs scrapper

readme.md

quick scrap docs on https://docs.ollama.com/

because of data been placed in monolitic repo instead of clone all data from https://github.com/ollama/ollama repository it is faster to download it by propably gitbooks property that allows us to request by knowing https://docs.ollama.com/quickstart url ask for https://docs.ollama.com/quickstart.md url. This with combining it with intel by quick peek on https://docs.ollama.com/robots.txt where sitemap is declared to reside at url https://docs.ollama.com/sitemap.xml.

REST is the best and SSH is the best too

openwebui-docs-grabbing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "211e442e-fde1-4fb0-8168-fb9b0d46db85",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import asyncio\n",
    "# from pyppeteer import launch\n",
    "\n",
    "# async def main():\n",
    "#     browser = await launch()\n",
    "#     page = await browser.newPage()\n",
    "#     await page.goto('https://docs.ollama.com/')\n",
    "#     await page.screenshot({'path': 'example.png'})\n",
    "#     await browser.close()\n",
    "\n",
    "# await main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "9d3259c4-229e-4477-9b60-479bcd4b29ee",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests in /opt/conda/lib/python3.13/site-packages (2.32.5)\n",
      "Requirement already satisfied: beautifulsoup4 in /opt/conda/lib/python3.13/site-packages (4.14.2)\n",
      "Requirement already satisfied: ipython in /opt/conda/lib/python3.13/site-packages (9.7.0)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.13/site-packages (2.3.3)\n",
      "Requirement already satisfied: marko in /opt/conda/lib/python3.13/site-packages (2.2.1)\n",
      "Requirement already satisfied: charset_normalizer<4,>=2 in /opt/conda/lib/python3.13/site-packages (from requests) (3.4.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.13/site-packages (from requests) (3.11)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.13/site-packages (from requests) (1.26.20)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.13/site-packages (from requests) (2025.11.12)\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/lib/python3.13/site-packages (from beautifulsoup4) (2.8)\n",
      "Requirement already satisfied: typing-extensions>=4.0.0 in /opt/conda/lib/python3.13/site-packages (from beautifulsoup4) (4.15.0)\n",
      "Requirement already satisfied: decorator>=4.3.2 in /opt/conda/lib/python3.13/site-packages (from ipython) (5.2.1)\n",
      "Requirement already satisfied: ipython-pygments-lexers>=1.0.0 in /opt/conda/lib/python3.13/site-packages (from ipython) (1.1.1)\n",
      "Requirement already satisfied: jedi>=0.18.1 in /opt/conda/lib/python3.13/site-packages (from ipython) (0.19.2)\n",
      "Requirement already satisfied: matplotlib-inline>=0.1.5 in /opt/conda/lib/python3.13/site-packages (from ipython) (0.2.1)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.13/site-packages (from ipython) (4.9.0)\n",
      "Requirement already satisfied: prompt_toolkit<3.1.0,>=3.0.41 in /opt/conda/lib/python3.13/site-packages (from ipython) (3.0.52)\n",
      "Requirement already satisfied: pygments>=2.11.0 in /opt/conda/lib/python3.13/site-packages (from ipython) (2.19.2)\n",
      "Requirement already satisfied: stack_data>=0.6.0 in /opt/conda/lib/python3.13/site-packages (from ipython) (0.6.3)\n",
      "Requirement already satisfied: traitlets>=5.13.0 in /opt/conda/lib/python3.13/site-packages (from ipython) (5.14.3)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.13/site-packages (from prompt_toolkit<3.1.0,>=3.0.41->ipython) (0.2.14)\n",
      "Requirement already satisfied: numpy>=1.26.0 in /opt/conda/lib/python3.13/site-packages (from pandas) (2.3.5)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.13/site-packages (from pandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.13/site-packages (from pandas) (2025.2)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.13/site-packages (from pandas) (2025.2)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /opt/conda/lib/python3.13/site-packages (from jedi>=0.18.1->ipython) (0.8.5)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.13/site-packages (from pexpect>4.3->ipython) (0.7.0)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.13/site-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n",
      "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython) (2.2.1)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython) (3.0.1)\n",
      "Requirement already satisfied: pure_eval in /opt/conda/lib/python3.13/site-packages (from stack_data>=0.6.0->ipython) (0.2.3)\n",
      "<module 'requests' from '/opt/conda/lib/python3.13/site-packages/requests/__init__.py'>\n",
      "<module 'bs4' from '/opt/conda/lib/python3.13/site-packages/bs4/__init__.py'>\n",
      "<module 'IPython' from '/opt/conda/lib/python3.13/site-packages/IPython/__init__.py'>\n",
      "<module 'pandas' from '/opt/conda/lib/python3.13/site-packages/pandas/__init__.py'>\n",
      "<module 'marko' from '/opt/conda/lib/python3.13/site-packages/marko/__init__.py'>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None, None, None]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!pip install requests beautifulsoup4 ipython pandas marko\n",
    "\n",
    "import requests\n",
    "import bs4\n",
    "import IPython \n",
    "import pandas \n",
    "import marko\n",
    "\n",
    "import inspect\n",
    "\n",
    "#[print(inspect.getsource(x)) for x in [requests, bs4, IPython, pandas, marko]]\n",
    "\n",
    "[print(x) for x in [requests, bs4, IPython, pandas, marko]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "c8437eb6-94a1-430a-9c6e-22ca112a13aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HTTP_GET: ok\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "http_get = None\n",
    "\n",
    "try:\n",
    "    http_get = requests.get(\"https://docs.ollama.com/sitemap.xml\")\n",
    "    print(\"HTTP_GET: ok\")\n",
    "except:\n",
    "    print(\"ERROR\")\n",
    "\n",
    "http_get\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "1b4714bb-2093-4c8a-b1f7-3a36f3916b2f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://docs.ollama.com/api-reference/get-version.md',\n",
       " 'https://docs.ollama.com/api-reference/show-model-details.md',\n",
       " 'https://docs.ollama.com/api/authentication.md',\n",
       " 'https://docs.ollama.com/api/chat.md',\n",
       " 'https://docs.ollama.com/api/copy.md',\n",
       " 'https://docs.ollama.com/api/create.md',\n",
       " 'https://docs.ollama.com/api/delete.md',\n",
       " 'https://docs.ollama.com/api/embed.md',\n",
       " 'https://docs.ollama.com/api/errors.md',\n",
       " 'https://docs.ollama.com/api/generate.md',\n",
       " 'https://docs.ollama.com/api/introduction.md',\n",
       " 'https://docs.ollama.com/api/openai-compatibility.md',\n",
       " 'https://docs.ollama.com/api/ps.md',\n",
       " 'https://docs.ollama.com/api/pull.md',\n",
       " 'https://docs.ollama.com/api/push.md',\n",
       " 'https://docs.ollama.com/api/streaming.md',\n",
       " 'https://docs.ollama.com/api/tags.md',\n",
       " 'https://docs.ollama.com/api/usage.md',\n",
       " 'https://docs.ollama.com/capabilities/embeddings.md',\n",
       " 'https://docs.ollama.com/capabilities/streaming.md',\n",
       " 'https://docs.ollama.com/capabilities/structured-outputs.md',\n",
       " 'https://docs.ollama.com/capabilities/thinking.md',\n",
       " 'https://docs.ollama.com/capabilities/tool-calling.md',\n",
       " 'https://docs.ollama.com/capabilities/vision.md',\n",
       " 'https://docs.ollama.com/capabilities/web-search.md',\n",
       " 'https://docs.ollama.com/cli.md',\n",
       " 'https://docs.ollama.com/cloud.md',\n",
       " 'https://docs.ollama.com/context-length.md',\n",
       " 'https://docs.ollama.com/docker.md',\n",
       " 'https://docs.ollama.com/faq.md',\n",
       " 'https://docs.ollama.com/gpu.md',\n",
       " 'https://docs.ollama.com/import.md',\n",
       " 'https://docs.ollama.com.md',\n",
       " 'https://docs.ollama.com/integrations/cline.md',\n",
       " 'https://docs.ollama.com/integrations/codex.md',\n",
       " 'https://docs.ollama.com/integrations/droid.md',\n",
       " 'https://docs.ollama.com/integrations/goose.md',\n",
       " 'https://docs.ollama.com/integrations/jetbrains.md',\n",
       " 'https://docs.ollama.com/integrations/n8n.md',\n",
       " 'https://docs.ollama.com/integrations/roo-code.md',\n",
       " 'https://docs.ollama.com/integrations/vscode.md',\n",
       " 'https://docs.ollama.com/integrations/xcode.md',\n",
       " 'https://docs.ollama.com/integrations/zed.md',\n",
       " 'https://docs.ollama.com/linux.md',\n",
       " 'https://docs.ollama.com/macos.md',\n",
       " 'https://docs.ollama.com/modelfile.md',\n",
       " 'https://docs.ollama.com/quickstart.md',\n",
       " 'https://docs.ollama.com/troubleshooting.md',\n",
       " 'https://docs.ollama.com/windows.md']"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md_list = [x.text+\".md\" for x in bs4.BeautifulSoup(http_get.text).find_all(\"loc\")]\n",
    "\n",
    "md_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "2a982a6c-3267-4bb5-924a-9da5a81ebc7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api-reference/get-version.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api-reference/show-model-details.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/authentication.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/chat.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/copy.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/create.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/delete.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/embed.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/errors.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/generate.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/introduction.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/openai-compatibility.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/ps.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/pull.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/push.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/streaming.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/tags.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/api/usage.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/capabilities/embeddings.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/capabilities/streaming.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/capabilities/structured-outputs.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/capabilities/thinking.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/capabilities/tool-calling.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/capabilities/vision.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/capabilities/web-search.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/cli.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/cloud.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/context-length.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/docker.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/faq.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/gpu.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/import.md\n",
      "ERROR\t FOR  https://docs.ollama.com.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/cline.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/codex.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/droid.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/goose.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/jetbrains.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/n8n.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/roo-code.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/vscode.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/xcode.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/integrations/zed.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/linux.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/macos.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/modelfile.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/quickstart.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/troubleshooting.md\n",
      "HTTP_GET:ok\t FOR  https://docs.ollama.com/windows.md\n"
     ]
    }
   ],
   "source": [
    "#base_url = 'https://docs.ollama.com.md/'\n",
    "\n",
    "error_collection = []\n",
    "md_responses = {}\n",
    "\n",
    "def http_getter(url):\n",
    "    global error_collection\n",
    "    result = None\n",
    "    try:\n",
    "        result = requests.get(url).text\n",
    "        print(\"HTTP_GET:ok\\t FOR \", url)\n",
    "    except:\n",
    "        print(\"ERROR\\t FOR \", url)\n",
    "        error_collection.append(url)\n",
    "    return result\n",
    "\n",
    "for x in md_list:\n",
    "    md_responses[ x ] = http_getter(x)\n",
    "\n",
    "#pandas.dataFrame(md_responses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "b12c38b9-f8e4-403c-95c0-efeeba5a2ec9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['https://docs.ollama.com/api-reference/get-version.md', 'https://docs.ollama.com/api-reference/show-model-details.md', 'https://docs.ollama.com/api/authentication.md', 'https://docs.ollama.com/api/chat.md', 'https://docs.ollama.com/api/copy.md', 'https://docs.ollama.com/api/create.md', 'https://docs.ollama.com/api/delete.md', 'https://docs.ollama.com/api/embed.md', 'https://docs.ollama.com/api/errors.md', 'https://docs.ollama.com/api/generate.md', 'https://docs.ollama.com/api/introduction.md', 'https://docs.ollama.com/api/openai-compatibility.md', 'https://docs.ollama.com/api/ps.md', 'https://docs.ollama.com/api/pull.md', 'https://docs.ollama.com/api/push.md', 'https://docs.ollama.com/api/streaming.md', 'https://docs.ollama.com/api/tags.md', 'https://docs.ollama.com/api/usage.md', 'https://docs.ollama.com/capabilities/embeddings.md', 'https://docs.ollama.com/capabilities/streaming.md', 'https://docs.ollama.com/capabilities/structured-outputs.md', 'https://docs.ollama.com/capabilities/thinking.md', 'https://docs.ollama.com/capabilities/tool-calling.md', 'https://docs.ollama.com/capabilities/vision.md', 'https://docs.ollama.com/capabilities/web-search.md', 'https://docs.ollama.com/cli.md', 'https://docs.ollama.com/cloud.md', 'https://docs.ollama.com/context-length.md', 'https://docs.ollama.com/docker.md', 'https://docs.ollama.com/faq.md', 'https://docs.ollama.com/gpu.md', 'https://docs.ollama.com/import.md', 'https://docs.ollama.com.md', 'https://docs.ollama.com/integrations/cline.md', 'https://docs.ollama.com/integrations/codex.md', 'https://docs.ollama.com/integrations/droid.md', 'https://docs.ollama.com/integrations/goose.md', 'https://docs.ollama.com/integrations/jetbrains.md', 'https://docs.ollama.com/integrations/n8n.md', 'https://docs.ollama.com/integrations/roo-code.md', 'https://docs.ollama.com/integrations/vscode.md', 'https://docs.ollama.com/integrations/xcode.md', 'https://docs.ollama.com/integrations/zed.md', 'https://docs.ollama.com/linux.md', 'https://docs.ollama.com/macos.md', 'https://docs.ollama.com/modelfile.md', 'https://docs.ollama.com/quickstart.md', 'https://docs.ollama.com/troubleshooting.md', 'https://docs.ollama.com/windows.md'])"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md_responses.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "6723b7d9-7ad5-403e-be87-604dad362051",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['api-reference/get-version.md',\n",
       " 'api-reference/show-model-details.md',\n",
       " 'api/authentication.md',\n",
       " 'api/chat.md',\n",
       " 'api/copy.md',\n",
       " 'api/create.md',\n",
       " 'api/delete.md',\n",
       " 'api/embed.md',\n",
       " 'api/errors.md',\n",
       " 'api/generate.md',\n",
       " 'api/introduction.md',\n",
       " 'api/openai-compatibility.md',\n",
       " 'api/ps.md',\n",
       " 'api/pull.md',\n",
       " 'api/push.md',\n",
       " 'api/streaming.md',\n",
       " 'api/tags.md',\n",
       " 'api/usage.md',\n",
       " 'capabilities/embeddings.md',\n",
       " 'capabilities/streaming.md',\n",
       " 'capabilities/structured-outputs.md',\n",
       " 'capabilities/thinking.md',\n",
       " 'capabilities/tool-calling.md',\n",
       " 'capabilities/vision.md',\n",
       " 'capabilities/web-search.md',\n",
       " 'cli.md',\n",
       " 'cloud.md',\n",
       " 'context-length.md',\n",
       " 'docker.md',\n",
       " 'faq.md',\n",
       " 'gpu.md',\n",
       " 'import.md',\n",
       " 'integrations/cline.md',\n",
       " 'integrations/codex.md',\n",
       " 'integrations/droid.md',\n",
       " 'integrations/goose.md',\n",
       " 'integrations/jetbrains.md',\n",
       " 'integrations/n8n.md',\n",
       " 'integrations/roo-code.md',\n",
       " 'integrations/vscode.md',\n",
       " 'integrations/xcode.md',\n",
       " 'integrations/zed.md',\n",
       " 'linux.md',\n",
       " 'macos.md',\n",
       " 'modelfile.md',\n",
       " 'quickstart.md',\n",
       " 'troubleshooting.md',\n",
       " 'windows.md']"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_url = 'https://docs.ollama.com'\n",
    "\n",
    "filepath_list = [x[len(base_url)+1:] for x in md_responses.keys() if x not in error_collection]\n",
    "\n",
    "filepath_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "ea16f98a-aef9-4445-afdb-caed8409d597",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('.')"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pathlib\n",
    "\n",
    "repo_path = pathlib.Path(\".\")\n",
    "\n",
    "repo_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "e83b49b9-4c1e-414b-a9b5-b0e450c1cef4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repo_path.exists()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "80162b15-6a93-42ca-8597-d2fefe268f0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/jovyan'"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(repo_path.resolve())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "69ab2d8b-4d2b-4a96-8f04-8c48e4cd3fe2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('api-reference/get-version.md')"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repo_path/filepath_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "3c804595-83ff-4250-ade8-21d36422097f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('.')"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repo_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "3b3dae15-f275-4e95-a3a6-c975f693da10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CREATE FOLDER\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "if not (repo_path/\"markdowned_docs.ollama.com\").exists():\n",
    "    os.mkdir(repo_path/\"markdown-gitbook_docs.ollama.com\")\n",
    "    print(\"CREATE FOLDER\")\n",
    "else:\n",
    "    print(\"FOLDER EXIST\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "7c373a47-fef5-455a-9edd-08ed98255361",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/jovyan/markdowned_docs.ollama.com/api-reference\n",
      "\n",
      "FILE -  api-reference/get-version.md  >>>  https://docs.ollama.com/api-reference/get-version.md\n",
      "\tIS EXIST\n",
      "/home/jovyan/markdowned_docs.ollama.com/api-reference\n",
      "\n",
      "FILE -  api-reference/show-model-details.md  >>>  https://docs.ollama.com/api-reference/show-model-details.md\n",
      "\t <class 'str'> 5229\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "dirmake\n",
      "\n",
      "FILE -  api/authentication.md  >>>  https://docs.ollama.com/api/authentication.md\n",
      "\t <class 'str'> 1871\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/chat.md  >>>  https://docs.ollama.com/api/chat.md\n",
      "\t <class 'str'> 23360\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/copy.md  >>>  https://docs.ollama.com/api/copy.md\n",
      "\t <class 'str'> 1643\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/create.md  >>>  https://docs.ollama.com/api/create.md\n",
      "\t <class 'str'> 5944\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/delete.md  >>>  https://docs.ollama.com/api/delete.md\n",
      "\t <class 'str'> 1387\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/embed.md  >>>  https://docs.ollama.com/api/embed.md\n",
      "\t <class 'str'> 6007\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/errors.md  >>>  https://docs.ollama.com/api/errors.md\n",
      "\t <class 'str'> 1602\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/generate.md  >>>  https://docs.ollama.com/api/generate.md\n",
      "\t <class 'str'> 19012\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/introduction.md  >>>  https://docs.ollama.com/api/introduction.md\n",
      "\t <class 'str'> 1437\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/openai-compatibility.md  >>>  https://docs.ollama.com/api/openai-compatibility.md\n",
      "\t <class 'str'> 22655\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/ps.md  >>>  https://docs.ollama.com/api/ps.md\n",
      "\t <class 'str'> 2547\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/pull.md  >>>  https://docs.ollama.com/api/pull.md\n",
      "\t <class 'str'> 3019\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/push.md  >>>  https://docs.ollama.com/api/push.md\n",
      "\t <class 'str'> 3062\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/streaming.md  >>>  https://docs.ollama.com/api/streaming.md\n",
      "\t <class 'str'> 1714\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/tags.md  >>>  https://docs.ollama.com/api/tags.md\n",
      "\t <class 'str'> 2983\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/api\n",
      "\n",
      "FILE -  api/usage.md  >>>  https://docs.ollama.com/api/usage.md\n",
      "\t <class 'str'> 1336\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/capabilities\n",
      "dirmake\n",
      "\n",
      "FILE -  capabilities/embeddings.md  >>>  https://docs.ollama.com/capabilities/embeddings.md\n",
      "\t <class 'str'> 3338\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/capabilities\n",
      "\n",
      "FILE -  capabilities/streaming.md  >>>  https://docs.ollama.com/capabilities/streaming.md\n",
      "\t <class 'str'> 3507\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/capabilities\n",
      "\n",
      "FILE -  capabilities/structured-outputs.md  >>>  https://docs.ollama.com/capabilities/structured-outputs.md\n",
      "\t <class 'str'> 5263\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/capabilities\n",
      "\n",
      "FILE -  capabilities/thinking.md  >>>  https://docs.ollama.com/capabilities/thinking.md\n",
      "\t <class 'str'> 4880\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/capabilities\n",
      "\n",
      "FILE -  capabilities/tool-calling.md  >>>  https://docs.ollama.com/capabilities/tool-calling.md\n",
      "\t <class 'str'> 23081\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/capabilities\n",
      "\n",
      "FILE -  capabilities/vision.md  >>>  https://docs.ollama.com/capabilities/vision.md\n",
      "\t <class 'str'> 2171\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/capabilities\n",
      "\n",
      "FILE -  capabilities/web-search.md  >>>  https://docs.ollama.com/capabilities/web-search.md\n",
      "\t <class 'str'> 16654\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  cli.md  >>>  https://docs.ollama.com/cli.md\n",
      "\t <class 'str'> 1276\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  cloud.md  >>>  https://docs.ollama.com/cloud.md\n",
      "\t <class 'str'> 5082\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  context-length.md  >>>  https://docs.ollama.com/context-length.md\n",
      "\t <class 'str'> 2742\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  docker.md  >>>  https://docs.ollama.com/docker.md\n",
      "\t <class 'str'> 2790\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  faq.md  >>>  https://docs.ollama.com/faq.md\n",
      "\t <class 'str'> 17701\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  gpu.md  >>>  https://docs.ollama.com/gpu.md\n",
      "\t <class 'str'> 11385\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  import.md  >>>  https://docs.ollama.com/import.md\n",
      "\t <class 'str'> 9006\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  md  >>>  https://docs.ollama.com.md\n",
      "\tIS EXIST\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "dirmake\n",
      "\n",
      "FILE -  integrations/cline.md  >>>  https://docs.ollama.com/integrations/cline.md\n",
      "\t <class 'str'> 2434\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/codex.md  >>>  https://docs.ollama.com/integrations/codex.md\n",
      "\t <class 'str'> 1175\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/droid.md  >>>  https://docs.ollama.com/integrations/droid.md\n",
      "\t <class 'str'> 1942\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/goose.md  >>>  https://docs.ollama.com/integrations/goose.md\n",
      "\t <class 'str'> 3927\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/jetbrains.md  >>>  https://docs.ollama.com/integrations/jetbrains.md\n",
      "\t <class 'str'> 5332\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/n8n.md  >>>  https://docs.ollama.com/integrations/n8n.md\n",
      "\t <class 'str'> 7124\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/roo-code.md  >>>  https://docs.ollama.com/integrations/roo-code.md\n",
      "\t <class 'str'> 1226\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/vscode.md  >>>  https://docs.ollama.com/integrations/vscode.md\n",
      "\t <class 'str'> 4804\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/xcode.md  >>>  https://docs.ollama.com/integrations/xcode.md\n",
      "\t <class 'str'> 5223\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com/integrations\n",
      "\n",
      "FILE -  integrations/zed.md  >>>  https://docs.ollama.com/integrations/zed.md\n",
      "\t <class 'str'> 3638\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  linux.md  >>>  https://docs.ollama.com/linux.md\n",
      "\t <class 'str'> 4601\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  macos.md  >>>  https://docs.ollama.com/macos.md\n",
      "\t <class 'str'> 1986\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  modelfile.md  >>>  https://docs.ollama.com/modelfile.md\n",
      "\t <class 'str'> 12981\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  quickstart.md  >>>  https://docs.ollama.com/quickstart.md\n",
      "\t <class 'str'> 2116\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  troubleshooting.md  >>>  https://docs.ollama.com/troubleshooting.md\n",
      "\t <class 'str'> 7368\n",
      "\t CREATE\n",
      "/home/jovyan/markdowned_docs.ollama.com\n",
      "\n",
      "FILE -  windows.md  >>>  https://docs.ollama.com/windows.md\n",
      "\t <class 'str'> 4423\n",
      "\t CREATE\n"
     ]
    }
   ],
   "source": [
    "base_url = 'https://docs.ollama.com'\n",
    "for fileurl in md_responses.keys():\n",
    "    filepath = fileurl[len(base_url)+1:]\n",
    "    fullpathfile = repo_path / \"markdowned_docs.ollama.com\" / filepath\n",
    "    print((fullpathfile/\"..\").resolve())\n",
    "    if not (fullpathfile/\"..\").resolve().exists():\n",
    "        print(\"dirmake\")\n",
    "        os.makedirs(str((fullpathfile/\"..\").resolve()))\n",
    "    print(\"\\nFILE - \", filepath, \" >>> \", fileurl)\n",
    "    if not fullpathfile.exists() and md_responses[fileurl] is not None:\n",
    "        with open(str(fullpathfile), \"w\") as md_newfile:\n",
    "            md_newfile.write( md_responses[fileurl] )\n",
    "        print(\"\\t\", type(md_responses[fileurl]), len(md_responses[fileurl]))\n",
    "        print(\"\\t\", \"CREATE\")\n",
    "    else:\n",
    "        print(\"\\tIS EXIST\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "34c7bcfc-d04e-4d11-90bf-618f88894e25",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('/home/jovyan/markdowned_docs.ollama.com',\n",
       "  ['api-reference', 'integrations', 'api', 'capabilities'],\n",
       "  ['docker.md',\n",
       "   'cloud.md',\n",
       "   'cli.md',\n",
       "   'quickstart.md',\n",
       "   'macos.md',\n",
       "   'modelfile.md',\n",
       "   'gpu.md',\n",
       "   'troubleshooting.md',\n",
       "   'linux.md',\n",
       "   'faq.md',\n",
       "   'import.md',\n",
       "   'context-length.md',\n",
       "   'windows.md']),\n",
       " ('/home/jovyan/markdowned_docs.ollama.com/api-reference',\n",
       "  ['get-version.md'],\n",
       "  ['show-model-details.md']),\n",
       " ('/home/jovyan/markdowned_docs.ollama.com/api-reference/get-version.md',\n",
       "  [],\n",
       "  []),\n",
       " ('/home/jovyan/markdowned_docs.ollama.com/integrations',\n",
       "  [],\n",
       "  ['jetbrains.md',\n",
       "   'n8n.md',\n",
       "   'xcode.md',\n",
       "   'droid.md',\n",
       "   'vscode.md',\n",
       "   'zed.md',\n",
       "   'codex.md',\n",
       "   'roo-code.md',\n",
       "   'cline.md',\n",
       "   'goose.md']),\n",
       " ('/home/jovyan/markdowned_docs.ollama.com/api',\n",
       "  [],\n",
       "  ['pull.md',\n",
       "   'introduction.md',\n",
       "   'tags.md',\n",
       "   'generate.md',\n",
       "   'chat.md',\n",
       "   'create.md',\n",
       "   'streaming.md',\n",
       "   'openai-compatibility.md',\n",
       "   'errors.md',\n",
       "   'embed.md',\n",
       "   'push.md',\n",
       "   'usage.md',\n",
       "   'ps.md',\n",
       "   'authentication.md',\n",
       "   'copy.md',\n",
       "   'delete.md']),\n",
       " ('/home/jovyan/markdowned_docs.ollama.com/capabilities',\n",
       "  [],\n",
       "  ['vision.md',\n",
       "   'streaming.md',\n",
       "   'thinking.md',\n",
       "   'web-search.md',\n",
       "   'embeddings.md',\n",
       "   'tool-calling.md',\n",
       "   'structured-outputs.md'])]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x for x in os.walk(str((repo_path/\"markdowned_docs.ollama.com\").resolve()))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "570da238-2b51-49a7-9410-ffc5f1f260ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "!python -m zipfile -c \"./markdown-docs-open-webui.zip\" ./markdowned_docs.ollama.com/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b69c8de8-587e-4edc-a699-2093a5568cce",
   "metadata": {},
   "source": [
    "# last cell creates for us an [ZIP FILE](/files/markdown-docs-open-webui.zip) to be downloaded.\n",
    "### that link to it only works after execution in local selfhosted jupyter lab instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee22b032-b90c-4dc6-9203-040bd0dbbeed",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!rm -rf ./markdown* # most lazy and crazy brutal! remember what that do and can do in linux environment\n",
    "# !sudo rm -rf /* # this one is only for most unresponsible usage - NEVER UNCOMMENT THIS LINE\n",
    "IPython.display("
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}

additional founding

• https://ollama-9269c548.mintlify.app/
  most commits are currently made by @mxyng so if mentioning works it will be seen... I think