From d8c7ea9e11a897d69a74329f89afbfa31456e5ac Mon Sep 17 00:00:00 2001 From: Manas Chopra Date: Sun, 26 Jul 2026 14:44:06 +0530 Subject: [PATCH] added module 2 notebook for cloud setup in beginner course --- course-beginners/getting_started.ipynb | 362 +++++++++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 course-beginners/getting_started.ipynb diff --git a/course-beginners/getting_started.ipynb b/course-beginners/getting_started.ipynb new file mode 100644 index 0000000..8d2282c --- /dev/null +++ b/course-beginners/getting_started.ipynb @@ -0,0 +1,362 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "qA82MEyvHV8M", + "outputId": "6d95b1e8-6f15-4f88-d8f0-f390e103639c" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: qdrant-client in /usr/local/lib/python3.12/dist-packages (1.18.0)\n", + "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.12/dist-packages (5.6.0)\n", + "Requirement already satisfied: grpcio>=1.41.0 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (1.82.1)\n", + "Requirement already satisfied: httpx>=0.20.0 in /usr/local/lib/python3.12/dist-packages (from httpx[http2]>=0.20.0->qdrant-client) (0.28.1)\n", + "Requirement already satisfied: numpy>=1.26 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (2.0.2)\n", + "Requirement already satisfied: portalocker<4.0,>=2.7.0 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (3.2.0)\n", + "Requirement already satisfied: protobuf>=3.20.0 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (5.29.6)\n", + "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (2.13.4)\n", + "Requirement already satisfied: urllib3<3,>=1.26.14 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (2.5.0)\n", + "Requirement already satisfied: transformers<6.0.0,>=4.41.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (5.13.1)\n", + "Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (1.23.0)\n", + "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (2.11.0+cpu)\n", + "Requirement already satisfied: scikit-learn>=0.22.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (1.6.1)\n", + "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (1.16.3)\n", + "Requirement already satisfied: typing_extensions>=4.5.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (4.16.0)\n", + "Requirement already satisfied: tqdm>=4.0.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (4.67.3)\n", + "Requirement already satisfied: anyio in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (4.14.2)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (2026.6.17)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (1.0.9)\n", + "Requirement already satisfied: idna in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (3.18)\n", + "Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (0.16.0)\n", + "Requirement already satisfied: h2<5,>=3 in /usr/local/lib/python3.12/dist-packages (from httpx[http2]>=0.20.0->qdrant-client) (4.3.0)\n", + "Requirement already satisfied: click<9.0.0,>=8.4.2 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (8.4.2)\n", + "Requirement already satisfied: filelock>=3.10.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (3.29.7)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (2025.3.0)\n", + "Requirement already satisfied: hf-xet<2.0.0,>=1.5.1 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (1.5.1)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (26.2)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (6.0.3)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (2.46.4)\n", + "Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.4.2)\n", + "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn>=0.22.0->sentence-transformers) (1.5.3)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn>=0.22.0->sentence-transformers) (3.6.0)\n", + "Requirement already satisfied: setuptools<82 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (75.2.0)\n", + "Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (1.14.0)\n", + "Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (3.6.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (3.1.6)\n", + "Requirement already satisfied: regex>=2025.10.22 in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (2025.11.3)\n", + "Requirement already satisfied: tokenizers<=0.23.0,>=0.22.0 in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (0.22.2)\n", + "Requirement already satisfied: typer in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (0.26.8)\n", + "Requirement already satisfied: safetensors>=0.8.0 in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (0.8.0)\n", + "Requirement already satisfied: hyperframe<7,>=6.1 in /usr/local/lib/python3.12/dist-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client) (6.1.0)\n", + "Requirement already satisfied: hpack<5,>=4.1 in /usr/local/lib/python3.12/dist-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client) (4.2.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch>=1.11.0->sentence-transformers) (1.3.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers) (3.0.3)\n", + "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (1.5.4)\n", + "Requirement already satisfied: rich>=13.8.0 in /usr/local/lib/python3.12/dist-packages (from typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (13.9.4)\n", + "Requirement already satisfied: annotated-doc>=0.0.2 in /usr/local/lib/python3.12/dist-packages (from typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (0.0.4)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (4.2.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (2.20.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich>=13.8.0->typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (0.1.2)\n" + ] + } + ], + "source": [ + "!pip install qdrant-client sentence-transformers" + ] + }, + { + "cell_type": "code", + "source": [ + "!pip install -q \"qdrant-client[fastembed]\"" + ], + "metadata": { + "id": "JlhMbb89a8Sx" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "from google.colab import userdata\n", + "key = userdata.get(\"QDRANT_API_KEY\")\n", + "print(\"Key loaded:\", bool(key), \"| length:\", len(key) if key else 0)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "zF1SD3ehNM8L", + "outputId": "ab08b035-a09c-4342-f9d4-1853d19552a2" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Key loaded: True | length: 176\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "from qdrant_client import QdrantClient\n", + "from google.colab import userdata\n", + "\n", + "client = QdrantClient(\n", + " url=\"https://735fd0e9-0ba8-4a2c-ab2c-d41e5bdb5640.eu-central-1-0.aws.cloud.qdrant.io:6333\",\n", + " api_key=userdata.get(\"QDRANT_API_KEY\"),\n", + ")\n", + "\n", + "print(client.get_collections()) # sanity check — should print collections=[]" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "xfnp7s4RNDlf", + "outputId": "cd408e44-3cf8-4cb3-8cf3-797624e75f51" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "collections=[CollectionDescription(name='aa'), CollectionDescription(name='article'), CollectionDescription(name='articles'), CollectionDescription(name='my_collection')]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Create the collection" + ], + "metadata": { + "id": "O3pJt-ioO4L4" + } + }, + { + "cell_type": "code", + "source": [ + "from qdrant_client import models\n", + "\n", + "client.create_collection(\n", + " collection_name=\"Article\",\n", + " vectors_config=models.VectorParams(\n", + " size=384, # matches all-MiniLM-L6-v2\n", + " distance=models.Distance.COSINE,\n", + " ),\n", + ")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZUzIi137O6Wq", + "outputId": "62536ca7-e471-466b-aa42-a1ebec97a01a" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": {}, + "execution_count": 6 + } + ] + }, + { + "cell_type": "code", + "source": [ + "print(client.get_collections()) # should now list 'articles'" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4xh2GIHsO8TZ", + "outputId": "b31456f4-c239-4255-b7a6-9a927589f7cc" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "collections=[CollectionDescription(name='Article'), CollectionDescription(name='aa'), CollectionDescription(name='article'), CollectionDescription(name='articles'), CollectionDescription(name='my_collection')]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Load embedding model and ingest data" + ], + "metadata": { + "id": "ghbFIFiQO5ul" + } + }, + { + "cell_type": "code", + "source": [ + "from qdrant_client import models\n", + "\n", + "documents = [\n", + " {\"id\": 1, \"text\": \"Car repair guide\", \"category\": \"automotive\"},\n", + " {\"id\": 2, \"text\": \"How to cook pasta\", \"category\": \"food\"},\n", + "]\n", + "\n", + "points = [\n", + " models.PointStruct(\n", + " id=doc[\"id\"],\n", + " vector=models.Document(text=doc[\"text\"], model=\"sentence-transformers/all-MiniLM-L6-v2\"),\n", + " payload={\"title\": doc[\"text\"], \"category\": doc[\"category\"]},\n", + " )\n", + " for doc in documents\n", + "]\n", + "\n", + "client.upload_points(collection_name=\"Article\", points=points)\n", + "print(\"Ingested\", len(points), \"points\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "DnPpceb6PBfX", + "outputId": "0e11d7a1-180a-42c9-c99f-41906054a4e6" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Ingested 2 points\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Create payload index" + ], + "metadata": { + "id": "WlQiQhHIPe3m" + } + }, + { + "cell_type": "code", + "source": [ + "client.create_payload_index(\n", + " collection_name=\"Article\",\n", + " field_name=\"category\",\n", + " field_schema=\"keyword\",\n", + ")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "y4XBnGvYPhDr", + "outputId": "6532a9b9-702f-4bd4-a392-95839ead5c6d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "UpdateResult(operation_id=3, status=)" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Query with a filter" + ], + "metadata": { + "id": "Et0oQN1WPEmX" + } + }, + { + "cell_type": "code", + "source": [ + "from qdrant_client import models\n", + "from qdrant_client.models import Filter, FieldCondition, MatchValue\n", + "\n", + "results = client.query_points(\n", + " collection_name=\"Article\", # lowercase, matches what you created\n", + " query=models.Document(\n", + " text=\"automobile maintenance\",\n", + " model=\"sentence-transformers/all-MiniLM-L6-v2\", # same model as ingestion\n", + " ),\n", + " query_filter=Filter(\n", + " must=[FieldCondition(key=\"category\", match=MatchValue(value=\"automotive\"))]\n", + " ),\n", + " limit=3,\n", + ")\n", + "\n", + "for r in results.points:\n", + " print(f\"Score: {r.score:.3f} | {r.payload['title']}\")" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6tCJf7NoPHD6", + "outputId": "4a214836-cfae-4714-c543-a299731b7acf" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Score: 0.590 | Car repair guide\n" + ] + } + ] + } + ] +} \ No newline at end of file