{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "79a31d92-c664-4ed9-8e8c-07b92d2c1680",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: docling in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (2.30.0)\n",
      "Requirement already satisfied: chromadb in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (1.0.5)\n",
      "Requirement already satisfied: sentence-transformers in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (4.1.0)\n",
      "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (4.12.3)\n",
      "Requirement already satisfied: certifi>=2024.7.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (2025.1.31)\n",
      "Requirement already satisfied: docling-core<3.0.0,>=2.26.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-core[chunking]<3.0.0,>=2.26.0->docling) (2.27.0)\n",
      "Requirement already satisfied: docling-ibm-models<4.0.0,>=3.4.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (3.4.1)\n",
      "Requirement already satisfied: docling-parse<5.0.0,>=4.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (4.0.1)\n",
      "Requirement already satisfied: easyocr<2.0,>=1.7 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (1.7.2)\n",
      "Requirement already satisfied: filetype<2.0.0,>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (1.2.0)\n",
      "Requirement already satisfied: huggingface_hub<1,>=0.23 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (0.30.2)\n",
      "Requirement already satisfied: lxml<6.0.0,>=4.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (5.3.2)\n",
      "Requirement already satisfied: marko<3.0.0,>=2.1.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (2.1.3)\n",
      "Requirement already satisfied: openpyxl<4.0.0,>=3.1.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (3.1.5)\n",
      "Requirement already satisfied: pandas<3.0.0,>=2.1.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (2.2.2)\n",
      "Requirement already satisfied: pillow<12.0.0,>=10.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (10.3.0)\n",
      "Requirement already satisfied: pluggy<2.0.0,>=1.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (1.5.0)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (2.11.3)\n",
      "Requirement already satisfied: pydantic-settings<3.0.0,>=2.3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (2.8.1)\n",
      "Requirement already satisfied: pylatexenc<3.0,>=2.10 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (2.10)\n",
      "Requirement already satisfied: pypdfium2<5.0.0,>=4.30.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (4.30.1)\n",
      "Requirement already satisfied: python-docx<2.0.0,>=1.1.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (1.1.2)\n",
      "Requirement already satisfied: python-pptx<2.0.0,>=1.0.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (1.0.2)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.32.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (2.32.2)\n",
      "Requirement already satisfied: rtree<2.0.0,>=1.3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (1.4.0)\n",
      "Requirement already satisfied: scipy<2.0.0,>=1.6.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (1.13.1)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.65.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (4.66.4)\n",
      "Requirement already satisfied: typer<0.16.0,>=0.12.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling) (0.15.2)\n",
      "Requirement already satisfied: build>=1.0.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.2.2.post1)\n",
      "Requirement already satisfied: chroma-hnswlib==0.7.6 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.7.6)\n",
      "Requirement already satisfied: fastapi==0.115.9 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.115.9)\n",
      "Requirement already satisfied: uvicorn>=0.18.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.34.1)\n",
      "Requirement already satisfied: numpy>=1.22.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (2.2.4)\n",
      "Requirement already satisfied: posthog>=2.4.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (3.25.0)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (4.13.2)\n",
      "Requirement already satisfied: onnxruntime>=1.14.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.21.0)\n",
      "Requirement already satisfied: opentelemetry-api>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.32.1)\n",
      "Requirement already satisfied: tokenizers>=0.13.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.21.1)\n",
      "Requirement already satisfied: pypika>=0.48.9 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.48.9)\n",
      "Requirement already satisfied: overrides>=7.3.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (7.7.0)\n",
      "Requirement already satisfied: importlib-resources in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (6.4.0)\n",
      "Requirement already satisfied: grpcio>=1.58.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.70.0)\n",
      "Requirement already satisfied: bcrypt>=4.0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (4.3.0)\n",
      "Requirement already satisfied: kubernetes>=28.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (32.0.1)\n",
      "Requirement already satisfied: tenacity>=8.2.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (9.1.2)\n",
      "Requirement already satisfied: pyyaml>=6.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (6.0.1)\n",
      "Requirement already satisfied: mmh3>=4.0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (5.1.0)\n",
      "Requirement already satisfied: orjson>=3.9.12 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (3.10.16)\n",
      "Requirement already satisfied: httpx>=0.27.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.27.0)\n",
      "Requirement already satisfied: rich>=10.11.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (13.9.4)\n",
      "Requirement already satisfied: jsonschema>=4.19.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (4.22.0)\n",
      "Requirement already satisfied: starlette<0.46.0,>=0.40.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from fastapi==0.115.9->chromadb) (0.45.3)\n",
      "Requirement already satisfied: transformers<5.0.0,>=4.41.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (4.51.3)\n",
      "Requirement already satisfied: torch>=1.11.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (2.6.0)\n",
      "Requirement already satisfied: scikit-learn in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (1.6.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from beautifulsoup4<5.0.0,>=4.12.3->docling) (2.5)\n",
      "Requirement already satisfied: packaging>=19.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from build>=1.0.3->chromadb) (24.0)\n",
      "Requirement already satisfied: pyproject_hooks in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from build>=1.0.3->chromadb) (1.2.0)\n",
      "Requirement already satisfied: jsonref<2.0.0,>=1.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-core<3.0.0,>=2.26.0->docling-core[chunking]<3.0.0,>=2.26.0->docling) (1.1.0)\n",
      "Requirement already satisfied: latex2mathml<4.0.0,>=3.77.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-core<3.0.0,>=2.26.0->docling-core[chunking]<3.0.0,>=2.26.0->docling) (3.77.0)\n",
      "Requirement already satisfied: tabulate<0.10.0,>=0.9.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-core<3.0.0,>=2.26.0->docling-core[chunking]<3.0.0,>=2.26.0->docling) (0.9.0)\n",
      "Requirement already satisfied: semchunk<3.0.0,>=2.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-core[chunking]<3.0.0,>=2.26.0->docling) (2.2.2)\n",
      "Requirement already satisfied: jsonlines<4.0.0,>=3.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-ibm-models<4.0.0,>=3.4.0->docling) (3.1.0)\n",
      "Requirement already satisfied: opencv-python-headless<5.0.0.0,>=4.6.0.66 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-ibm-models<4.0.0,>=3.4.0->docling) (4.11.0.86)\n",
      "Requirement already satisfied: safetensors<1,>=0.4.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from safetensors[torch]<1,>=0.4.3->docling-ibm-models<4.0.0,>=3.4.0->docling) (0.5.3)\n",
      "Requirement already satisfied: torchvision<1,>=0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from docling-ibm-models<4.0.0,>=3.4.0->docling) (0.21.0)\n",
      "Requirement already satisfied: scikit-image in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from easyocr<2.0,>=1.7->docling) (0.25.2)\n",
      "Requirement already satisfied: python-bidi in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from easyocr<2.0,>=1.7->docling) (0.6.6)\n",
      "Requirement already satisfied: Shapely in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from easyocr<2.0,>=1.7->docling) (2.1.0)\n",
      "Requirement already satisfied: pyclipper in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from easyocr<2.0,>=1.7->docling) (1.3.0.post6)\n",
      "Requirement already satisfied: ninja in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from easyocr<2.0,>=1.7->docling) (1.11.1.4)\n",
      "Requirement already satisfied: anyio in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (4.3.0)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (1.0.5)\n",
      "Requirement already satisfied: idna in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (3.7)\n",
      "Requirement already satisfied: sniffio in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.27.0->chromadb) (0.14.0)\n",
      "Requirement already satisfied: filelock in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from huggingface_hub<1,>=0.23->docling) (3.18.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from huggingface_hub<1,>=0.23->docling) (2025.3.2)\n",
      "Requirement already satisfied: attrs>=22.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (23.2.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (2023.12.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (0.35.1)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (0.18.1)\n",
      "Requirement already satisfied: six>=1.9.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.9.0)\n",
      "Requirement already satisfied: google-auth>=1.0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.39.0)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n",
      "Requirement already satisfied: requests-oauthlib in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\n",
      "Requirement already satisfied: oauthlib>=3.2.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
      "Requirement already satisfied: urllib3>=1.24.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: durationpy>=0.7 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (0.9)\n",
      "Requirement already satisfied: coloredlogs in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
      "Requirement already satisfied: flatbuffers in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (25.2.10)\n",
      "Requirement already satisfied: protobuf in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (5.29.3)\n",
      "Requirement already satisfied: sympy in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (1.13.1)\n",
      "Requirement already satisfied: et-xmlfile in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from openpyxl<4.0.0,>=3.1.5->docling) (2.0.0)\n",
      "Requirement already satisfied: deprecated>=1.2.6 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.18)\n",
      "Requirement already satisfied: importlib-metadata<8.7.0,>=6.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-api>=1.2.0->chromadb) (7.1.0)\n",
      "Requirement already satisfied: googleapis-common-protos~=1.52 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.70.0)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.32.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-proto==1.32.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-instrumentation==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-semantic-conventions==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-util-http==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation==0.53b1->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.17.2)\n",
      "Requirement already satisfied: asgiref~=3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-asgi==0.53b1->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (3.8.1)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pandas<3.0.0,>=2.1.4->docling) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pandas<3.0.0,>=2.1.4->docling) (2024.1)\n",
      "Requirement already satisfied: monotonic>=1.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: distro>=1.5.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from posthog>=2.4.0->chromadb) (1.9.0)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->docling) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.33.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->docling) (2.33.1)\n",
      "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.0.0->docling) (0.4.0)\n",
      "Requirement already satisfied: python-dotenv>=0.21.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic-settings<3.0.0,>=2.3.0->docling) (1.1.0)\n",
      "Requirement already satisfied: XlsxWriter>=0.5.7 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from python-pptx<2.0.0,>=1.0.2->docling) (3.2.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests<3.0.0,>=2.32.2->docling) (3.3.2)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from rich>=10.11.0->chromadb) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from rich>=10.11.0->chromadb) (2.18.0)\n",
      "Requirement already satisfied: networkx in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (3.4.2)\n",
      "Requirement already satisfied: jinja2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (3.1.4)\n",
      "Requirement already satisfied: setuptools in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (70.0.0)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (2024.11.6)\n",
      "Requirement already satisfied: click>=8.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from typer<0.16.0,>=0.12.5->docling) (8.1.8)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from typer<0.16.0,>=0.12.5->docling) (1.5.4)\n",
      "Requirement already satisfied: httptools>=0.6.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.4)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.21.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.5)\n",
      "Requirement already satisfied: websockets>=10.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (15.0.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from scikit-learn->sentence-transformers) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from scikit-learn->sentence-transformers) (3.5.0)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.5.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.2)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from importlib-metadata<8.7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->chromadb) (0.1.2)\n",
      "Requirement already satisfied: mpire[dill] in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from semchunk<3.0.0,>=2.2.0->docling-core[chunking]<3.0.0,>=2.26.0->docling) (2.10.2)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jinja2->torch>=1.11.0->sentence-transformers) (2.1.5)\n",
      "Requirement already satisfied: imageio!=2.35.0,>=2.33 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling) (2.37.0)\n",
      "Requirement already satisfied: tifffile>=2022.8.12 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling) (2025.3.30)\n",
      "Requirement already satisfied: lazy-loader>=0.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from scikit-image->easyocr<2.0,>=1.7->docling) (0.4)\n",
      "Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.1)\n",
      "Requirement already satisfied: multiprocess>=0.70.15 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from mpire[dill]->semchunk<3.0.0,>=2.2.0->docling-core[chunking]<3.0.0,>=2.26.0->docling) (0.70.18)\n",
      "Requirement already satisfied: dill>=0.4.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from multiprocess>=0.70.15->mpire[dill]->semchunk<3.0.0,>=2.2.0->docling-core[chunking]<3.0.0,>=2.26.0->docling) (0.4.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install docling chromadb sentence-transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "76269d1d-17c5-470c-a5c2-cff095cd1e53",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pymupdf in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (1.25.5)\n",
      "Requirement already satisfied: tqdm in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (4.66.4)\n",
      "Requirement already satisfied: spacy in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (3.8.5)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (3.0.12)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (1.0.5)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (1.0.12)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (2.0.11)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (3.0.9)\n",
      "Requirement already satisfied: thinc<8.4.0,>=8.3.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (8.3.6)\n",
      "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (1.1.3)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (2.5.1)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (2.0.10)\n",
      "Requirement already satisfied: weasel<0.5.0,>=0.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (0.4.1)\n",
      "Requirement already satisfied: typer<1.0.0,>=0.3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (0.15.2)\n",
      "Requirement already satisfied: numpy>=1.19.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (2.2.4)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (2.32.2)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (2.11.3)\n",
      "Requirement already satisfied: jinja2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (3.1.4)\n",
      "Requirement already satisfied: setuptools in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (70.0.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (24.0)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from spacy) (3.5.0)\n",
      "Requirement already satisfied: language-data>=1.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from langcodes<4.0.0,>=3.2.0->spacy) (1.3.0)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.33.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (2.33.1)\n",
      "Requirement already satisfied: typing-extensions>=4.12.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (4.13.2)\n",
      "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (0.4.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests<3.0.0,>=2.13.0->spacy) (2025.1.31)\n",
      "Requirement already satisfied: blis<1.4.0,>=1.3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from thinc<8.4.0,>=8.3.4->spacy) (1.3.0)\n",
      "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from thinc<8.4.0,>=8.3.4->spacy) (0.1.5)\n",
      "Requirement already satisfied: click>=8.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from typer<1.0.0,>=0.3.0->spacy) (8.1.8)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from typer<1.0.0,>=0.3.0->spacy) (1.5.4)\n",
      "Requirement already satisfied: rich>=10.11.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from typer<1.0.0,>=0.3.0->spacy) (13.9.4)\n",
      "Requirement already satisfied: cloudpathlib<1.0.0,>=0.7.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from weasel<0.5.0,>=0.1.0->spacy) (0.21.0)\n",
      "Requirement already satisfied: smart-open<8.0.0,>=5.2.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from weasel<0.5.0,>=0.1.0->spacy) (7.1.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jinja2->spacy) (2.1.5)\n",
      "Requirement already satisfied: marisa-trie>=1.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy) (1.2.1)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy) (2.18.0)\n",
      "Requirement already satisfied: wrapt in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from smart-open<8.0.0,>=5.2.1->weasel<0.5.0,>=0.1.0->spacy) (1.17.2)\n",
      "Requirement already satisfied: mdurl~=0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer<1.0.0,>=0.3.0->spacy) (0.1.2)\n",
      "Collecting it-core-news-sm==3.8.0\n",
      "  Using cached https://github.com/explosion/spacy-models/releases/download/it_core_news_sm-3.8.0/it_core_news_sm-3.8.0-py3-none-any.whl (13.0 MB)\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('it_core_news_sm')\n"
     ]
    }
   ],
   "source": [
    "!pip install pymupdf tqdm spacy\n",
    "!python -m spacy download it_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6c7eb59b-6ec2-45e3-8c9c-81b744f97785",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (4.51.3)\n",
      "Requirement already satisfied: filelock in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (3.18.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (0.30.2)\n",
      "Requirement already satisfied: numpy>=1.17 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (2.2.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (24.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (2024.11.6)\n",
      "Requirement already satisfied: requests in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (2.32.2)\n",
      "Requirement already satisfied: tokenizers<0.22,>=0.21 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (0.21.1)\n",
      "Requirement already satisfied: safetensors>=0.4.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (0.5.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers) (4.66.4)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (2025.3.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (4.13.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests->transformers) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests->transformers) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests->transformers) (2.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests->transformers) (2025.1.31)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "481ee40f-059f-4feb-ae74-044413d75438",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae9a7fe2333e435ead670beabaccec4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "📄 Lettura pagine PDF:   0%|          | 0/66 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import fitz  # PyMuPDF\n",
    "from tqdm.auto import tqdm\n",
    "import pandas as pd\n",
    "\n",
    "def text_formatter(text: str) -> str:\n",
    "    # Pulizia semplice\n",
    "    import re\n",
    "    text = text.replace(\"\\n\", \" \").strip()\n",
    "    text = re.sub(r\"[ \\t]{2,}\", \" \", text)\n",
    "    text = re.sub(r\"\\.{2,}\", \" \", text)  # sostituisce ... con spazio\n",
    "    text = re.sub(r\"Pagina\\s+\\d+\\s+di\\s+\\d+\", \"\", text, flags=re.IGNORECASE)\n",
    "    text = re.sub(r\"Creazione VM su Cloud INSIEL\",\"\", text)\n",
    "    text = re.sub(r\"IO_XX_00_XX  ISTRUZIONE OPERATIVA 22/10/2024\", \"\",text)\n",
    "    text = re.sub(r\"IO_XX_00_XX  ISTRUZIONE OPERATIVA 22/10/2024 \",\"\",text)\n",
    "    return text.strip()\n",
    "\n",
    "def open_and_read_pdf(pdf_path: str):\n",
    "    doc = fitz.open(pdf_path)\n",
    "    pages = []\n",
    "    for page_number, page in tqdm(enumerate(doc), total=len(doc), desc=\"📄 Lettura pagine PDF\"):\n",
    "        text = text_formatter(page.get_text())\n",
    "        pages.append({\n",
    "            \"page_number\": page_number + 1,\n",
    "            \"page_char_count\": len(text),\n",
    "            \"page_word_count\": len(text.split()),\n",
    "            \"page_token_estimate\": len(text) // 4,\n",
    "            \"text\": text\n",
    "        })\n",
    "    return pages\n",
    "\n",
    "pdf_path = \"data/insiel.pdf\"  # Cambia se il tuo file è altrove\n",
    "pages_and_texts = open_and_read_pdf(pdf_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c723bd2b-a58b-45b4-b480-fcbf4b7214ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5728eb3cb94f4eca944908b941ce7ece",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "✂️ Split in frasi:   0%|          | 0/66 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import spacy\n",
    "nlp = spacy.load(\"it_core_news_sm\")\n",
    "\n",
    "# Spezza il testo di ogni pagina in frasi\n",
    "for page in tqdm(pages_and_texts, desc=\"✂️ Split in frasi\"):\n",
    "    doc = nlp(page[\"text\"])\n",
    "    sentences = [sent.text.strip() for sent in doc.sents if sent.text.strip()]\n",
    "    page[\"sentence_chunks\"] = []\n",
    "\n",
    "    CHUNK_SIZE = 10  # Gruppi da 5 frasi\n",
    "    for i in range(0, len(sentences), CHUNK_SIZE):\n",
    "        chunk = sentences[i:i + CHUNK_SIZE]\n",
    "        page[\"sentence_chunks\"].append(chunk)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bd7f511f-cdd9-48ae-b97b-0fb3b878bb35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'page_number': 66,\n",
       " 'page_char_count': 45,\n",
       " 'page_word_count': 9,\n",
       " 'page_token_estimate': 11,\n",
       " 'text': 'FIGURA 88 - VMWARE NSX - CRITERIO GRUPPO   58',\n",
       " 'sentence_chunks': [['FIGURA 88 - VMWARE NSX - CRITERIO GRUPPO   58']]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pages_and_texts[65]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a6b65f41-9187-4667-b272-8f0117ee60b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page_number</th>\n",
       "      <th>page_char_count</th>\n",
       "      <th>page_word_count</th>\n",
       "      <th>page_token_estimate</th>\n",
       "      <th>text</th>\n",
       "      <th>sentence_chunks</th>\n",
       "      <th>chunk_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>62</td>\n",
       "      <td>1301</td>\n",
       "      <td>173</td>\n",
       "      <td>325</td>\n",
       "      <td>7 APPENDICE 2 – NAMING CONVENTION 7.1 NOMI MAC...</td>\n",
       "      <td>[[7 APPENDICE 2 – NAMING CONVENTION 7.1 NOMI M...</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>63</td>\n",
       "      <td>344</td>\n",
       "      <td>19</td>\n",
       "      <td>86</td>\n",
       "      <td>9.3 VMWARE ARIA • HTTPS://DOCS.VMWARE.COM/IT/V...</td>\n",
       "      <td>[[9.3 VMWARE ARIA • HTTPS://DOCS.VMWARE.COM/IT...</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>64</td>\n",
       "      <td>1949</td>\n",
       "      <td>352</td>\n",
       "      <td>487</td>\n",
       "      <td>12 INDICE DELLE FIGURE FIGURA 1 - VMWARE VSPHE...</td>\n",
       "      <td>[[12 INDICE DELLE FIGURE FIGURA 1 - VMWARE VSP...</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>65</td>\n",
       "      <td>2293</td>\n",
       "      <td>446</td>\n",
       "      <td>573</td>\n",
       "      <td>FIGURA 43 – RESOURCES – MENU CONTESTUALE   30 ...</td>\n",
       "      <td>[[FIGURA 43 – RESOURCES – MENU CONTESTUALE   3...</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>66</td>\n",
       "      <td>45</td>\n",
       "      <td>9</td>\n",
       "      <td>11</td>\n",
       "      <td>FIGURA 88 - VMWARE NSX - CRITERIO GRUPPO   58</td>\n",
       "      <td>[[FIGURA 88 - VMWARE NSX - CRITERIO GRUPPO   58]]</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    page_number  page_char_count  page_word_count  page_token_estimate  \\\n",
       "61           62             1301              173                  325   \n",
       "62           63              344               19                   86   \n",
       "63           64             1949              352                  487   \n",
       "64           65             2293              446                  573   \n",
       "65           66               45                9                   11   \n",
       "\n",
       "                                                 text  \\\n",
       "61  7 APPENDICE 2 – NAMING CONVENTION 7.1 NOMI MAC...   \n",
       "62  9.3 VMWARE ARIA • HTTPS://DOCS.VMWARE.COM/IT/V...   \n",
       "63  12 INDICE DELLE FIGURE FIGURA 1 - VMWARE VSPHE...   \n",
       "64  FIGURA 43 – RESOURCES – MENU CONTESTUALE   30 ...   \n",
       "65      FIGURA 88 - VMWARE NSX - CRITERIO GRUPPO   58   \n",
       "\n",
       "                                      sentence_chunks chunk_id  \n",
       "61  [[7 APPENDICE 2 – NAMING CONVENTION 7.1 NOMI M...       61  \n",
       "62  [[9.3 VMWARE ARIA • HTTPS://DOCS.VMWARE.COM/IT...       62  \n",
       "63  [[12 INDICE DELLE FIGURE FIGURA 1 - VMWARE VSP...       63  \n",
       "64  [[FIGURA 43 – RESOURCES – MENU CONTESTUALE   3...       64  \n",
       "65  [[FIGURA 88 - VMWARE NSX - CRITERIO GRUPPO   58]]       65  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(pages_and_texts)\n",
    "df[\"chunk_id\"] = df.index.astype(str)\n",
    "\n",
    "# Mostra i primi\n",
    "df.tail()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b88c68ab-0fe8-407c-999d-46bccf95c95b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(66, 7)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8926ba95-94a7-4ae1-a217-718bc84918a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "page_number            9\n",
       "page_char_count        9\n",
       "page_word_count        9\n",
       "page_token_estimate    9\n",
       "text                   9\n",
       "sentence_chunks        9\n",
       "chunk_id               9\n",
       "dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['page_token_estimate'] < 60].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b5c7ffd4-55a6-4486-ac70-1f5678f72046",
   "metadata": {},
   "outputs": [],
   "source": [
    "final = df[df['page_token_estimate'] > 60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8e1fc3ca-a065-4c32-9acb-6cc300c7d2a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>page_number</th>\n",
       "      <th>page_char_count</th>\n",
       "      <th>page_word_count</th>\n",
       "      <th>page_token_estimate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>57.00</td>\n",
       "      <td>57.00</td>\n",
       "      <td>57.00</td>\n",
       "      <td>57.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>34.67</td>\n",
       "      <td>821.02</td>\n",
       "      <td>129.28</td>\n",
       "      <td>204.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>18.73</td>\n",
       "      <td>463.38</td>\n",
       "      <td>79.10</td>\n",
       "      <td>115.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.00</td>\n",
       "      <td>245.00</td>\n",
       "      <td>19.00</td>\n",
       "      <td>61.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>19.00</td>\n",
       "      <td>457.00</td>\n",
       "      <td>71.00</td>\n",
       "      <td>114.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>35.00</td>\n",
       "      <td>733.00</td>\n",
       "      <td>115.00</td>\n",
       "      <td>183.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>51.00</td>\n",
       "      <td>1067.00</td>\n",
       "      <td>164.00</td>\n",
       "      <td>266.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>65.00</td>\n",
       "      <td>2293.00</td>\n",
       "      <td>446.00</td>\n",
       "      <td>573.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       page_number  page_char_count  page_word_count  page_token_estimate\n",
       "count        57.00            57.00            57.00                57.00\n",
       "mean         34.67           821.02           129.28               204.91\n",
       "std          18.73           463.38            79.10               115.84\n",
       "min           3.00           245.00            19.00                61.00\n",
       "25%          19.00           457.00            71.00               114.00\n",
       "50%          35.00           733.00           115.00               183.00\n",
       "75%          51.00          1067.00           164.00               266.00\n",
       "max          65.00          2293.00           446.00               573.00"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.describe().round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d48140df-d520-48f7-b3c2-f4ceb3346150",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sentence-transformers in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (4.1.0)\n",
      "Requirement already satisfied: chromadb in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (1.0.5)\n",
      "Requirement already satisfied: transformers<5.0.0,>=4.41.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (4.51.3)\n",
      "Requirement already satisfied: tqdm in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (4.66.4)\n",
      "Requirement already satisfied: torch>=1.11.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (2.6.0)\n",
      "Requirement already satisfied: scikit-learn in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (1.6.1)\n",
      "Requirement already satisfied: scipy in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (1.13.1)\n",
      "Requirement already satisfied: huggingface-hub>=0.20.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (0.30.2)\n",
      "Requirement already satisfied: Pillow in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (10.3.0)\n",
      "Requirement already satisfied: typing_extensions>=4.5.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sentence-transformers) (4.13.2)\n",
      "Requirement already satisfied: build>=1.0.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.2.2.post1)\n",
      "Requirement already satisfied: pydantic>=1.9 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (2.11.3)\n",
      "Requirement already satisfied: chroma-hnswlib==0.7.6 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.7.6)\n",
      "Requirement already satisfied: fastapi==0.115.9 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.115.9)\n",
      "Requirement already satisfied: uvicorn>=0.18.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.34.1)\n",
      "Requirement already satisfied: numpy>=1.22.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (2.2.4)\n",
      "Requirement already satisfied: posthog>=2.4.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (3.25.0)\n",
      "Requirement already satisfied: onnxruntime>=1.14.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.21.0)\n",
      "Requirement already satisfied: opentelemetry-api>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.32.1)\n",
      "Requirement already satisfied: tokenizers>=0.13.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.21.1)\n",
      "Requirement already satisfied: pypika>=0.48.9 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.48.9)\n",
      "Requirement already satisfied: overrides>=7.3.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (7.7.0)\n",
      "Requirement already satisfied: importlib-resources in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (6.4.0)\n",
      "Requirement already satisfied: grpcio>=1.58.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (1.70.0)\n",
      "Requirement already satisfied: bcrypt>=4.0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (4.3.0)\n",
      "Requirement already satisfied: typer>=0.9.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.15.2)\n",
      "Requirement already satisfied: kubernetes>=28.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (32.0.1)\n",
      "Requirement already satisfied: tenacity>=8.2.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (9.1.2)\n",
      "Requirement already satisfied: pyyaml>=6.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (6.0.1)\n",
      "Requirement already satisfied: mmh3>=4.0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (5.1.0)\n",
      "Requirement already satisfied: orjson>=3.9.12 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (3.10.16)\n",
      "Requirement already satisfied: httpx>=0.27.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (0.27.0)\n",
      "Requirement already satisfied: rich>=10.11.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (13.9.4)\n",
      "Requirement already satisfied: jsonschema>=4.19.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from chromadb) (4.22.0)\n",
      "Requirement already satisfied: starlette<0.46.0,>=0.40.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from fastapi==0.115.9->chromadb) (0.45.3)\n",
      "Requirement already satisfied: packaging>=19.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from build>=1.0.3->chromadb) (24.0)\n",
      "Requirement already satisfied: pyproject_hooks in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from build>=1.0.3->chromadb) (1.2.0)\n",
      "Requirement already satisfied: anyio in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (4.3.0)\n",
      "Requirement already satisfied: certifi in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (2025.1.31)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (1.0.5)\n",
      "Requirement already satisfied: idna in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (3.7)\n",
      "Requirement already satisfied: sniffio in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpx>=0.27.0->chromadb) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.27.0->chromadb) (0.14.0)\n",
      "Requirement already satisfied: filelock in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from huggingface-hub>=0.20.0->sentence-transformers) (3.18.0)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from huggingface-hub>=0.20.0->sentence-transformers) (2025.3.2)\n",
      "Requirement already satisfied: requests in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from huggingface-hub>=0.20.0->sentence-transformers) (2.32.2)\n",
      "Requirement already satisfied: attrs>=22.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (23.2.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (2023.12.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (0.35.1)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jsonschema>=4.19.0->chromadb) (0.18.1)\n",
      "Requirement already satisfied: six>=1.9.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.9.0)\n",
      "Requirement already satisfied: google-auth>=1.0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.39.0)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n",
      "Requirement already satisfied: requests-oauthlib in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\n",
      "Requirement already satisfied: oauthlib>=3.2.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
      "Requirement already satisfied: urllib3>=1.24.2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: durationpy>=0.7 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from kubernetes>=28.1.0->chromadb) (0.9)\n",
      "Requirement already satisfied: coloredlogs in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
      "Requirement already satisfied: flatbuffers in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (25.2.10)\n",
      "Requirement already satisfied: protobuf in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (5.29.3)\n",
      "Requirement already satisfied: sympy in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from onnxruntime>=1.14.1->chromadb) (1.13.1)\n",
      "Requirement already satisfied: deprecated>=1.2.6 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.18)\n",
      "Requirement already satisfied: importlib-metadata<8.7.0,>=6.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-api>=1.2.0->chromadb) (7.1.0)\n",
      "Requirement already satisfied: googleapis-common-protos~=1.52 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.70.0)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.32.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-proto==1.32.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.32.1)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-instrumentation==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-semantic-conventions==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: opentelemetry-util-http==0.53b1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.53b1)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation==0.53b1->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.17.2)\n",
      "Requirement already satisfied: asgiref~=3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from opentelemetry-instrumentation-asgi==0.53b1->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (3.8.1)\n",
      "Requirement already satisfied: monotonic>=1.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: distro>=1.5.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from posthog>=2.4.0->chromadb) (1.9.0)\n",
      "Requirement already satisfied: annotated-types>=0.6.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic>=1.9->chromadb) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.33.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic>=1.9->chromadb) (2.33.1)\n",
      "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pydantic>=1.9->chromadb) (0.4.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from rich>=10.11.0->chromadb) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from rich>=10.11.0->chromadb) (2.18.0)\n",
      "Requirement already satisfied: networkx in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (3.4.2)\n",
      "Requirement already satisfied: jinja2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (3.1.4)\n",
      "Requirement already satisfied: setuptools in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from torch>=1.11.0->sentence-transformers) (70.0.0)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (2024.11.6)\n",
      "Requirement already satisfied: safetensors>=0.4.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from transformers<5.0.0,>=4.41.0->sentence-transformers) (0.5.3)\n",
      "Requirement already satisfied: click>=8.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from typer>=0.9.0->chromadb) (8.1.8)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from typer>=0.9.0->chromadb) (1.5.4)\n",
      "Requirement already satisfied: httptools>=0.6.3 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.4)\n",
      "Requirement already satisfied: python-dotenv>=0.13 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.1.0)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.21.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.5)\n",
      "Requirement already satisfied: websockets>=10.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (15.0.1)\n",
      "Requirement already satisfied: joblib>=1.2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from scikit-learn->sentence-transformers) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=3.1.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from scikit-learn->sentence-transformers) (3.5.0)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.5.2)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.2)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from importlib-metadata<8.7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->chromadb) (0.1.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from requests->huggingface-hub>=0.20.0->sentence-transformers) (3.3.2)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from jinja2->torch>=1.11.0->sentence-transformers) (2.1.5)\n",
      "Requirement already satisfied: pyasn1<0.7.0,>=0.6.1 in /Users/alegio98/Library/jupyterlab-desktop/jlab_server/lib/python3.12/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install sentence-transformers chromadb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d80d1aec-bb7c-491e-87e5-622ffbdb29b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b03b146f585f450db759a0d35e257b62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "embedding_model = SentenceTransformer(\"sentence-transformers/distiluse-base-multilingual-cased-v1\")\n",
    "\n",
    "texts = final[\"text\"].tolist()\n",
    "chunk_ids = final[\"chunk_id\"].tolist()\n",
    "metadatas = [{\"page\": int(p)} for p in final[\"page_number\"]]\n",
    "\n",
    "embeddings = embedding_model.encode(texts, show_progress_bar=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "27cb5630-e180-4c66-9c02-05b797e14c6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import chromadb\n",
    "\n",
    "# nuovo client\n",
    "client = chromadb.PersistentClient(path=\"./vectorstore\")\n",
    "\n",
    "# collection\n",
    "collection = client.get_or_create_collection(\"insiel_chunks\")\n",
    "\n",
    "# aggiunta\n",
    "collection.add(\n",
    "    documents=texts,\n",
    "    embeddings=embeddings.tolist(),\n",
    "    metadatas=metadatas,\n",
    "    ids=chunk_ids\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6d1f5c63-7d31-49a3-bb15-9d681a22b47e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Domanda:  ciao\n"
     ]
    }
   ],
   "source": [
    "query = input(\"Domanda: \")\n",
    "query_embedding = embedding_model.encode([query])\n",
    "\n",
    "results = collection.query(\n",
    "    query_embeddings=query_embedding,\n",
    "    n_results=3  # puoi aumentare a 5, 10, ecc.\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1c5dde05-99a5-4ee9-ad9a-112888745464",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "🔹 RISULTATO 1 (pagina 41):\n",
      "o 17 macchine virtuali (vedi PAR. 5.4.1.1.4.4). o 0 profili di accesso a Livello 7 (vedi PAR. 5.4.1.1.4.3). o 2 sevizi (vedi PAR 5.4.1.1.4.1) o 6 profili contesto (vedi PAR. 5.4.1.1.4.3). 5.4.1.1.2.2 AVVISI Nel tab ‘Avvisi’ sono presenti due sotto tab: • Avvisi • Definizione avvisi Figura 60 - VMWare NSX – Home – Avvisi Figura 61 - VMWare NSX – Home – Definizione avvisi...\n",
      "---\n",
      "\n",
      "🔹 RISULTATO 2 (pagina 60):\n",
      "5.5.3.1.1 MICROSOFT WINDOWS SERVER 5.5.3.1.2 LINUX 5.5.3.2 AGGIUNTA A DOMINIO 5.5.3.3 ESPANSIONE DISCO DI SISTEMA 5.5.3.4 AGGIORNAMENTO SISTEMA OPERATIVO 5.5.3.4.1 VERIFICA SU CONSOLE WSUS 5.5.3.4.2 AGGIORNAMENTO E PATCHING 5.5.3.5 INSTALLAZIONE ANTIVIRUS 5.5.3.5.1 VERIFICA FUNZIONAMENTO SU CONSOLE ANTIVIRUS...\n",
      "---\n",
      "\n",
      "🔹 RISULTATO 3 (pagina 3):\n",
      "INDICE 1 SCOPO   5 2 AMBITO DI APPLICAZIONE  5 3 ACRONIMI E DEFINIZIONI   5 4 RESPONSABILITA’   5 5 MODALITÀ ESECUTIVE   6 5.1 CONFIGURAZIONE DELL’INFRASTRUTTURA  6 5.1.1 VMWARE VSPHERE   6 5.1.2 VMWARE NSX   7 5.1.3 VMWARE ARIA   8 5.2 IMPLEMENTAZIONE DELL’INFRASTRUTTURA   9 5.2.1 LOCALIZZAZIONE   9 5.2.2 CONNETTIVITA’ DI RETE   9 5.2.2.1 CONNETTIVITA’ FISICA   9 5.2.2.2 CONNETTIVITA’ LOGICA   9 5.3 CLOUD INSIEL   11 5.3.1 ACCESSO AL CLOUD INSIEL   11 5.3.1.1 LOGIN   11 5.3.1.2 RESET PASSWORD  ...\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "for i, (doc, meta) in enumerate(zip(results[\"documents\"][0], results[\"metadatas\"][0])):\n",
    "    print(f\"\\n🔹 RISULTATO {i+1} (pagina {meta['page']}):\")\n",
    "    print(doc[:500] + \"...\\n---\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "065512aa-94b1-4d68-82c9-3ab3b35fb534",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Device set to use cpu\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline\n",
    "\n",
    "model_id = \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
    "model = AutoModelForCausalLM.from_pretrained(model_id).to(torch.device(\"cpu\"))\n",
    "\n",
    "rag_chat = pipeline(\"text-generation\", model=model, tokenizer=tokenizer, max_new_tokens=200, device=-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef436727-c660-4943-be68-f3c02d998a91",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_rag_response_local(query, retrieved_chunks):\n",
    "    context = \"\\n\\n\".join(retrieved_chunks)\n",
    "    \n",
    "    prompt = f\"\"\"[INST] Usa solo le informazioni fornite nel contesto qui sotto per rispondere alla domanda, la risposta deve finire sempre con un punto. \n",
    "Se la risposta non è presente, di' chiaramente che non è specificato nel documento.\n",
    "\n",
    "Contesto:\n",
    "{context}\n",
    "\n",
    "Domanda: {query}\n",
    "Risposta: [/INST]\n",
    "\"\"\"\n",
    "    result = rag_chat(prompt)[0][\"generated_text\"]\n",
    "    return result.split(\"Risposta:\")[-1].strip()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bde08911-05ae-4c87-8892-2747b87d89fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 🧠 Inserisci la domanda\n",
    "query = input(\"Domanda: \")\n",
    "\n",
    "# 🔎 Ottieni l'embedding della query (usa sentence-transformers, NON il modello generativo!)\n",
    "query_embedding = embedding_model.encode([query])  \n",
    "\n",
    "# 🔍 Retrieval dei chunk più simili da Chroma\n",
    "results = collection.query(\n",
    "    query_embeddings=query_embedding,\n",
    "    n_results=3\n",
    ")\n",
    "\n",
    "# 🧱 Estrai i chunk di contesto\n",
    "retrieved_chunks = results[\"documents\"][0]\n",
    "\n",
    "# 🤖 Genera la risposta usando il modello open-source locale\n",
    "response = generate_rag_response_local(query, retrieved_chunks)\n",
    "\n",
    "# 🖨️ Mostra la risposta\n",
    "print(\"🤖 Risposta:\\n\", response)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12423be3-2773-4246-bcc8-d98634638cc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "retrieved_chunks\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88f9cd0b-c6b7-491e-8be3-6f76eb2b435d",
   "metadata": {},
   "outputs": [],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b8b383a1-6630-488b-908c-70da2975ca8a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}