Skip to content

added module 2 notebook for cloud setup in beginner course #114

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
manas95826 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

added module 2 notebook for cloud setup in beginner course #114

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
362 changes: 362 additions & 0 deletions course-beginners/getting_started.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,362 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"collapsed": true,
"id": "qA82MEyvHV8M",
"outputId": "6d95b1e8-6f15-4f88-d8f0-f390e103639c"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Requirement already satisfied: qdrant-client in /usr/local/lib/python3.12/dist-packages (1.18.0)\n",
"Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.12/dist-packages (5.6.0)\n",
"Requirement already satisfied: grpcio>=1.41.0 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (1.82.1)\n",
"Requirement already satisfied: httpx>=0.20.0 in /usr/local/lib/python3.12/dist-packages (from httpx[http2]>=0.20.0->qdrant-client) (0.28.1)\n",
"Requirement already satisfied: numpy>=1.26 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (2.0.2)\n",
"Requirement already satisfied: portalocker<4.0,>=2.7.0 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (3.2.0)\n",
"Requirement already satisfied: protobuf>=3.20.0 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (5.29.6)\n",
"Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (2.13.4)\n",
"Requirement already satisfied: urllib3<3,>=1.26.14 in /usr/local/lib/python3.12/dist-packages (from qdrant-client) (2.5.0)\n",
"Requirement already satisfied: transformers<6.0.0,>=4.41.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (5.13.1)\n",
"Requirement already satisfied: huggingface-hub>=0.23.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (1.23.0)\n",
"Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (2.11.0+cpu)\n",
"Requirement already satisfied: scikit-learn>=0.22.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (1.6.1)\n",
"Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (1.16.3)\n",
"Requirement already satisfied: typing_extensions>=4.5.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (4.16.0)\n",
"Requirement already satisfied: tqdm>=4.0.0 in /usr/local/lib/python3.12/dist-packages (from sentence-transformers) (4.67.3)\n",
"Requirement already satisfied: anyio in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (4.14.2)\n",
"Requirement already satisfied: certifi in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (2026.6.17)\n",
"Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (1.0.9)\n",
"Requirement already satisfied: idna in /usr/local/lib/python3.12/dist-packages (from httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (3.18)\n",
"Requirement already satisfied: h11>=0.16 in /usr/local/lib/python3.12/dist-packages (from httpcore==1.*->httpx>=0.20.0->httpx[http2]>=0.20.0->qdrant-client) (0.16.0)\n",
"Requirement already satisfied: h2<5,>=3 in /usr/local/lib/python3.12/dist-packages (from httpx[http2]>=0.20.0->qdrant-client) (4.3.0)\n",
"Requirement already satisfied: click<9.0.0,>=8.4.2 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (8.4.2)\n",
"Requirement already satisfied: filelock>=3.10.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (3.29.7)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (2025.3.0)\n",
"Requirement already satisfied: hf-xet<2.0.0,>=1.5.1 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (1.5.1)\n",
"Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (26.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.12/dist-packages (from huggingface-hub>=0.23.0->sentence-transformers) (6.0.3)\n",
"Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.12/dist-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.7.0)\n",
"Requirement already satisfied: pydantic-core==2.46.4 in /usr/local/lib/python3.12/dist-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (2.46.4)\n",
"Requirement already satisfied: typing-inspection>=0.4.2 in /usr/local/lib/python3.12/dist-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.4.2)\n",
"Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn>=0.22.0->sentence-transformers) (1.5.3)\n",
"Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.12/dist-packages (from scikit-learn>=0.22.0->sentence-transformers) (3.6.0)\n",
"Requirement already satisfied: setuptools<82 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (75.2.0)\n",
"Requirement already satisfied: sympy>=1.13.3 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (1.14.0)\n",
"Requirement already satisfied: networkx>=2.5.1 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (3.6.1)\n",
"Requirement already satisfied: jinja2 in /usr/local/lib/python3.12/dist-packages (from torch>=1.11.0->sentence-transformers) (3.1.6)\n",
"Requirement already satisfied: regex>=2025.10.22 in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (2025.11.3)\n",
"Requirement already satisfied: tokenizers<=0.23.0,>=0.22.0 in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (0.22.2)\n",
"Requirement already satisfied: typer in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (0.26.8)\n",
"Requirement already satisfied: safetensors>=0.8.0 in /usr/local/lib/python3.12/dist-packages (from transformers<6.0.0,>=4.41.0->sentence-transformers) (0.8.0)\n",
"Requirement already satisfied: hyperframe<7,>=6.1 in /usr/local/lib/python3.12/dist-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client) (6.1.0)\n",
"Requirement already satisfied: hpack<5,>=4.1 in /usr/local/lib/python3.12/dist-packages (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client) (4.2.0)\n",
"Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.12/dist-packages (from sympy>=1.13.3->torch>=1.11.0->sentence-transformers) (1.3.0)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.12/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers) (3.0.3)\n",
"Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.12/dist-packages (from typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (1.5.4)\n",
"Requirement already satisfied: rich>=13.8.0 in /usr/local/lib/python3.12/dist-packages (from typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (13.9.4)\n",
"Requirement already satisfied: annotated-doc>=0.0.2 in /usr/local/lib/python3.12/dist-packages (from typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (0.0.4)\n",
"Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (4.2.0)\n",
"Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.12/dist-packages (from rich>=13.8.0->typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (2.20.0)\n",
"Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.12/dist-packages (from markdown-it-py>=2.2.0->rich>=13.8.0->typer->transformers<6.0.0,>=4.41.0->sentence-transformers) (0.1.2)\n"
]
}
],
"source": [
"!pip install qdrant-client sentence-transformers"
]
},
{
"cell_type": "code",
"source": [
"!pip install -q \"qdrant-client[fastembed]\""
],
"metadata": {
"id": "JlhMbb89a8Sx"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"from google.colab import userdata\n",
"key = userdata.get(\"QDRANT_API_KEY\")\n",
"print(\"Key loaded:\", bool(key), \"| length:\", len(key) if key else 0)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"collapsed": true,
"id": "zF1SD3ehNM8L",
"outputId": "ab08b035-a09c-4342-f9d4-1853d19552a2"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Key loaded: True | length: 176\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"from qdrant_client import QdrantClient\n",
"from google.colab import userdata\n",
"\n",
"client = QdrantClient(\n",
" url=\"https://735fd0e9-0ba8-4a2c-ab2c-d41e5bdb5640.eu-central-1-0.aws.cloud.qdrant.io:6333\",\n",
" api_key=userdata.get(\"QDRANT_API_KEY\"),\n",
")\n",
"\n",
"print(client.get_collections()) # sanity check — should print collections=[]"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "xfnp7s4RNDlf",
"outputId": "cd408e44-3cf8-4cb3-8cf3-797624e75f51"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"collections=[CollectionDescription(name='aa'), CollectionDescription(name='article'), CollectionDescription(name='articles'), CollectionDescription(name='my_collection')]\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Create the collection"
],
"metadata": {
"id": "O3pJt-ioO4L4"
}
},
{
"cell_type": "code",
"source": [
"from qdrant_client import models\n",
"\n",
"client.create_collection(\n",
" collection_name=\"Article\",\n",
" vectors_config=models.VectorParams(\n",
" size=384, # matches all-MiniLM-L6-v2\n",
" distance=models.Distance.COSINE,\n",
" ),\n",
")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ZUzIi137O6Wq",
"outputId": "62536ca7-e471-466b-aa42-a1ebec97a01a"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"True"
]
},
"metadata": {},
"execution_count": 6
}
]
},
{
"cell_type": "code",
"source": [
"print(client.get_collections()) # should now list 'articles'"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "4xh2GIHsO8TZ",
"outputId": "b31456f4-c239-4255-b7a6-9a927589f7cc"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"collections=[CollectionDescription(name='Article'), CollectionDescription(name='aa'), CollectionDescription(name='article'), CollectionDescription(name='articles'), CollectionDescription(name='my_collection')]\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Load embedding model and ingest data"
],
"metadata": {
"id": "ghbFIFiQO5ul"
}
},
{
"cell_type": "code",
"source": [
"from qdrant_client import models\n",
"\n",
"documents = [\n",
" {\"id\": 1, \"text\": \"Car repair guide\", \"category\": \"automotive\"},\n",
" {\"id\": 2, \"text\": \"How to cook pasta\", \"category\": \"food\"},\n",
"]\n",
"\n",
"points = [\n",
" models.PointStruct(\n",
" id=doc[\"id\"],\n",
" vector=models.Document(text=doc[\"text\"], model=\"sentence-transformers/all-MiniLM-L6-v2\"),\n",
" payload={\"title\": doc[\"text\"], \"category\": doc[\"category\"]},\n",
" )\n",
" for doc in documents\n",
"]\n",
"\n",
"client.upload_points(collection_name=\"Article\", points=points)\n",
"print(\"Ingested\", len(points), \"points\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "DnPpceb6PBfX",
"outputId": "0e11d7a1-180a-42c9-c99f-41906054a4e6"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Ingested 2 points\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Create payload index"
],
"metadata": {
"id": "WlQiQhHIPe3m"
}
},
{
"cell_type": "code",
"source": [
"client.create_payload_index(\n",
" collection_name=\"Article\",\n",
" field_name=\"category\",\n",
" field_schema=\"keyword\",\n",
")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "y4XBnGvYPhDr",
"outputId": "6532a9b9-702f-4bd4-a392-95839ead5c6d"
},
"execution_count": null,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"UpdateResult(operation_id=3, status=<UpdateStatus.COMPLETED: 'completed'>)"
]
},
"metadata": {},
"execution_count": 10
}
]
},
{
"cell_type": "markdown",
"source": [
"# Query with a filter"
],
"metadata": {
"id": "Et0oQN1WPEmX"
}
},
{
"cell_type": "code",
"source": [
"from qdrant_client import models\n",
"from qdrant_client.models import Filter, FieldCondition, MatchValue\n",
"\n",
"results = client.query_points(\n",
" collection_name=\"Article\", # lowercase, matches what you created\n",
" query=models.Document(\n",
" text=\"automobile maintenance\",\n",
" model=\"sentence-transformers/all-MiniLM-L6-v2\", # same model as ingestion\n",
" ),\n",
" query_filter=Filter(\n",
" must=[FieldCondition(key=\"category\", match=MatchValue(value=\"automotive\"))]\n",
" ),\n",
" limit=3,\n",
")\n",
"\n",
"for r in results.points:\n",
" print(f\"Score: {r.score:.3f} | {r.payload['title']}\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "6tCJf7NoPHD6",
"outputId": "4a214836-cfae-4714-c543-a299731b7acf"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Score: 0.590 | Car repair guide\n"
]
}
]
}
]
}