diff --git a/docs/examples/classify_job.md b/docs/examples/classify_job.md index 5f7dd807d..4415ece55 100644 --- a/docs/examples/classify_job.md +++ b/docs/examples/classify_job.md @@ -2,46 +2,112 @@ To learn about the basics of creating a job, please refer to the [quickstart guide](../quickstart.md). -In this example, we want to rate different images based on a Likert scale to assess how well generated images match their descriptions. The `NoShuffleSetting` setting ensures answer options remain in order since they represent a scale. - -```python -from rapidata import RapidataClient, NoShuffleSetting - -IMAGE_URLS = [ - "https://assets.rapidata.ai/tshirt-4o.png", - "https://assets.rapidata.ai/tshirt-aurora.jpg", - "https://assets.rapidata.ai/teamleader-aurora.jpg", -] - -CONTEXTS = ["A t-shirt with the text 'Running on caffeine & dreams'"] * len(IMAGE_URLS) - -client = RapidataClient() - -audience = client.audience.create_audience(name="Likert Scale Audience") -audience.add_classification_example( - instruction="How well does the image match the description?", - answer_options=["1: Not at all", "2: A little", "3: Moderately", "4: Very well", "5: Perfectly"], - datapoint="https://assets.rapidata.ai/tshirt-4o.png", - truth=["5: Perfectly"], - context="A t-shirt with the text 'Running on caffeine & dreams'" -) - -job_definition = client.job.create_classification_job_definition( - name="Likert Scale Example", - instruction="How well does the image match the description?", - answer_options=["1: Not at all", "2: A little", "3: Moderately", "4: Very well", "5: Perfectly"], - contexts=CONTEXTS, - datapoints=IMAGE_URLS, - responses_per_datapoint=25, - settings=[NoShuffleSetting()] # (1)! -) - -job_definition.preview() - -job = audience.assign_job(job_definition) -job.display_progress_bar() -results = job.get_results() -print(results) -``` - -1. Keeps the answer options in their specified order. Without this, options are randomized to reduce bias — but for Likert scales you want them ordered. +In this example, we rate images on a Likert scale to assess how well generated images match their descriptions. The `NoShuffleSetting` keeps the answer options in order, since they represent a scale. + +=== "Simple" + + The simple version runs straight away on a **curated** audience — a pre-existing pool of trained labelers — so the job starts collecting responses immediately. + + ```python + from rapidata import RapidataClient, NoShuffleSetting + + IMAGE_URLS = [ + "https://assets.rapidata.ai/tshirt-4o.png", + "https://assets.rapidata.ai/tshirt-aurora.jpg", + "https://assets.rapidata.ai/teamleader-aurora.jpg", + ] + + CONTEXTS = ["A t-shirt with the text 'Running on caffeine & dreams'"] * len(IMAGE_URLS) + + client = RapidataClient() + + audience = client.audience.get_audience_by_id("aud_mr3NbeWa4Uo") # (1)! + + job_definition = client.job.create_classification_job_definition( + name="Likert Scale Example", + instruction="How well does the image match the description?", + answer_options=["1: Not at all", "2: A little", "3: Moderately", "4: Very well", "5: Perfectly"], + contexts=CONTEXTS, + datapoints=IMAGE_URLS, + responses_per_datapoint=25, + settings=[NoShuffleSetting()] # (2)! + ) + + job_definition.preview() + + job = audience.assign_job(job_definition) + job.display_progress_bar() + results = job.get_results() + print(results) + ``` + + 1. Looks up the curated **Coherence** audience by id, which already has trained labelers. A freshly created audience has no qualified labelers yet, so a job assigned to it would never collect responses — see the Advanced tab for how to build and train your own. You can browse the curated audiences and copy their ids from the [Rapidata Dashboard](https://app.rapidata.ai/audiences). + 2. Keeps the answer options in their specified order. Without this, options are randomized to reduce bias — but for Likert scales you want them ordered. + +=== "Advanced" + + The advanced version builds a **custom** audience and trains labelers with qualification examples before running the job. Only labelers who answer the examples correctly join the audience, which raises label quality on nuanced tasks. + + !!! warning "This takes significantly longer" + Unlike the Simple path, this first builds and trains an entirely new audience before the job can start collecting responses — expect it to take considerably longer to return results. + + ```python + from rapidata import RapidataClient, NoShuffleSetting + + IMAGE_URLS = [ + "https://assets.rapidata.ai/tshirt-4o.png", + "https://assets.rapidata.ai/tshirt-aurora.jpg", + "https://assets.rapidata.ai/teamleader-aurora.jpg", + ] + + CONTEXTS = ["A t-shirt with the text 'Running on caffeine & dreams'"] * len(IMAGE_URLS) + + ANSWER_OPTIONS = ["1: Not at all", "2: A little", "3: Moderately", "4: Very well", "5: Perfectly"] + + # Qualification examples — each pairs an image with a description and the + # correct rating. Use only examples whose truth is clear and unambiguous. + EXAMPLES = [ + ("https://assets.rapidata.ai/tshirt-4o.png", "A t-shirt with the text 'Running on caffeine & dreams'", "5: Perfectly"), + ("https://assets.rapidata.ai/flux_duck.jpg", "A psychedelic duck with glasses", "5: Perfectly"), + ("https://assets.rapidata.ai/flux_flower.jpg", "A yellow flower sticking out of a green pot", "5: Perfectly"), + ("https://assets.rapidata.ai/teamleader-aurora.jpg", "A t-shirt with the text 'Running on caffeine & dreams'", "1: Not at all"), + ("https://assets.rapidata.ai/flux_book.jpg", "A psychedelic duck with glasses", "1: Not at all"), + ("https://assets.rapidata.ai/flux_duck.jpg", "A small blue book sitting on a large red book", "1: Not at all"), + ] + + client = RapidataClient() + + audience = client.audience.create_audience(name="Likert Scale Audience") # (1)! + for datapoint, context, truth in EXAMPLES: + audience.add_classification_example( + instruction="How well does the image match the description?", + answer_options=ANSWER_OPTIONS, + datapoint=datapoint, + truth=[truth], + context=context, + settings=[NoShuffleSetting()] # (2)! + ) + + job_definition = client.job.create_classification_job_definition( + name="Likert Scale Example", + instruction="How well does the image match the description?", + answer_options=ANSWER_OPTIONS, + contexts=CONTEXTS, + datapoints=IMAGE_URLS, + responses_per_datapoint=25, + settings=[NoShuffleSetting()] + ) + + job_definition.preview() + + job = audience.assign_job(job_definition) + job.display_progress_bar() + results = job.get_results() + print(results) + ``` + + 1. Creates a new, empty audience. The `add_classification_example` calls below define who qualifies to join it. + 2. Qualify labelers on the same UI they'll see in the job. Since the job uses `NoShuffleSetting`, the examples use it too — see [Custom Audiences](../audiences.md#matching-the-job-ui-with-settings). + + !!! note + Review every qualification example and its truth carefully, and add more than the few shown here for production workloads — see [Custom Audiences](../audiences.md) for the full guide. diff --git a/docs/examples/compare_job.md b/docs/examples/compare_job.md index a0e3d722e..8fe9de3ee 100644 --- a/docs/examples/compare_job.md +++ b/docs/examples/compare_job.md @@ -4,50 +4,128 @@ To learn about the basics of creating a job, please refer to the [quickstart gui In this example, we compare images from two image generation models (Flux and Midjourney) to determine which more accurately follows the given prompts. -```python -from rapidata import RapidataClient - -PROMPTS = [ - "A sign that says 'Diffusion'.", - "A yellow flower sticking out of a green pot.", - "hyperrealism render of a surreal alien humanoid.", - "psychedelic duck", - "A small blue book sitting on a large red book." -] - -IMAGE_PAIRS = [ - ["https://assets.rapidata.ai/flux_sign_diffusion.jpg", "https://assets.rapidata.ai/mj_sign_diffusion.jpg"], - ["https://assets.rapidata.ai/flux_flower.jpg", "https://assets.rapidata.ai/mj_flower.jpg"], - ["https://assets.rapidata.ai/flux_alien.jpg", "https://assets.rapidata.ai/mj_alien.jpg"], - ["https://assets.rapidata.ai/flux_duck.jpg", "https://assets.rapidata.ai/mj_duck.jpg"], - ["https://assets.rapidata.ai/flux_book.jpg", "https://assets.rapidata.ai/mj_book.jpg"] -] - -client = RapidataClient() - -audience = client.audience.create_audience(name="Prompt Alignment Audience") -audience.add_compare_example( - instruction="Which image follows the prompt more accurately?", - datapoint=[ - "https://assets.rapidata.ai/flux_sign_diffusion.jpg", - "https://assets.rapidata.ai/mj_sign_diffusion.jpg" - ], - truth="https://assets.rapidata.ai/flux_sign_diffusion.jpg", - context="A sign that says 'Diffusion'." -) - -job_definition = client.job.create_compare_job_definition( - name="Example Image Prompt Alignment Job", - instruction="Which image follows the prompt more accurately?", - datapoints=IMAGE_PAIRS, - responses_per_datapoint=25, - contexts=PROMPTS -) - -job_definition.preview() - -job = audience.assign_job(job_definition) -job.display_progress_bar() -results = job.get_results() -print(results) -``` +=== "Simple" + + The simple version runs straight away on a **curated** audience — a pre-existing pool of trained labelers — so the job starts collecting responses immediately. + + ```python + from rapidata import RapidataClient + + PROMPTS = [ + "A sign that says 'Diffusion'.", + "A yellow flower sticking out of a green pot.", + "hyperrealism render of a surreal alien humanoid.", + "psychedelic duck", + "A small blue book sitting on a large red book." + ] + + IMAGE_PAIRS = [ + ["https://assets.rapidata.ai/flux_sign_diffusion.jpg", "https://assets.rapidata.ai/mj_sign_diffusion.jpg"], + ["https://assets.rapidata.ai/flux_flower.jpg", "https://assets.rapidata.ai/mj_flower.jpg"], + ["https://assets.rapidata.ai/flux_alien.jpg", "https://assets.rapidata.ai/mj_alien.jpg"], + ["https://assets.rapidata.ai/flux_duck.jpg", "https://assets.rapidata.ai/mj_duck.jpg"], + ["https://assets.rapidata.ai/flux_book.jpg", "https://assets.rapidata.ai/mj_book.jpg"] + ] + + client = RapidataClient() + + audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO") # (1)! + + job_definition = client.job.create_compare_job_definition( + name="Example Image Prompt Alignment Job", + instruction="Which image follows the prompt more accurately?", + datapoints=IMAGE_PAIRS, + responses_per_datapoint=25, + contexts=PROMPTS + ) + + job_definition.preview() + + job = audience.assign_job(job_definition) + job.display_progress_bar() + results = job.get_results() + print(results) + ``` + + 1. Looks up the curated **Alignment** audience by id, which already has trained labelers. A freshly created audience has no qualified labelers yet, so a job assigned to it would never collect responses — see the Advanced tab for how to build and train your own. You can browse the curated audiences and copy their ids from the [Rapidata Dashboard](https://app.rapidata.ai/audiences). + +=== "Advanced" + + The advanced version builds a **custom** audience and trains labelers with qualification examples before running the job. Only labelers who pick the correct image on the examples join the audience, which raises label quality. + + !!! warning "This takes significantly longer" + Unlike the Simple path, this first builds and trains an entirely new audience before the job can start collecting responses — expect it to take considerably longer to return results. + + ```python + from rapidata import RapidataClient + + PROMPTS = [ + "A sign that says 'Diffusion'.", + "A yellow flower sticking out of a green pot.", + "hyperrealism render of a surreal alien humanoid.", + "psychedelic duck", + "A small blue book sitting on a large red book." + ] + + IMAGE_PAIRS = [ + ["https://assets.rapidata.ai/flux_sign_diffusion.jpg", "https://assets.rapidata.ai/mj_sign_diffusion.jpg"], + ["https://assets.rapidata.ai/flux_flower.jpg", "https://assets.rapidata.ai/mj_flower.jpg"], + ["https://assets.rapidata.ai/flux_alien.jpg", "https://assets.rapidata.ai/mj_alien.jpg"], + ["https://assets.rapidata.ai/flux_duck.jpg", "https://assets.rapidata.ai/mj_duck.jpg"], + ["https://assets.rapidata.ai/flux_book.jpg", "https://assets.rapidata.ai/mj_book.jpg"] + ] + + # Qualification pairs where the first (Flux) image clearly follows the prompt + # better. The truth must point at the unambiguously better image. + QUALIFICATION_PAIRS = [ + ["https://assets.rapidata.ai/flux_sign_diffusion.jpg", "https://assets.rapidata.ai/mj_sign_diffusion.jpg"], + ["https://assets.rapidata.ai/flux_duck.jpg", "https://assets.rapidata.ai/mj_duck.jpg"], + ["https://assets.rapidata.ai/flux_book.jpg", "https://assets.rapidata.ai/mj_book.jpg"], + ["https://assets.rapidata.ai/flux_flower.jpg", "https://assets.rapidata.ai/mj_flower.jpg"], + ["https://assets.rapidata.ai/flux_store_front.jpg", "https://assets.rapidata.ai/mj_store_front.jpg"], + ["https://assets.rapidata.ai/flux_hand.jpg", "https://assets.rapidata.ai/mj_hand.jpg"], + ["https://assets.rapidata.ai/flux_traffic_lights.jpg", "https://assets.rapidata.ai/mj_traffic_lights.jpg"], + ["https://assets.rapidata.ai/flux_plane.jpg", "https://assets.rapidata.ai/mj_plane.jpg"], + ] + QUALIFICATION_PROMPTS = [ + "A sign that says 'Diffusion'.", + "A psychedelic duck with glasses", + "A small blue book sitting on a large red book.", + "A yellow flower sticking out of a bright green pot.", + "A store front with 'hello world' written on it.", + "A yellow hand on a black stone.", + "A green, yellow and red traffic light.", + "A plane flying over a person.", + ] + + client = RapidataClient() + + audience = client.audience.create_audience(name="Custom Prompt Alignment Audience") # (1)! + for prompt, datapoint in zip(QUALIFICATION_PROMPTS, QUALIFICATION_PAIRS): + audience.add_compare_example( + instruction="Which image follows the prompt more accurately?", + datapoint=datapoint, + truth=datapoint[0], + context=prompt + ) + + job_definition = client.job.create_compare_job_definition( + name="Example Image Prompt Alignment Job", + instruction="Which image follows the prompt more accurately?", + datapoints=IMAGE_PAIRS, + responses_per_datapoint=25, + contexts=PROMPTS + ) + + job_definition.preview() + + job = audience.assign_job(job_definition) + job.display_progress_bar() + results = job.get_results() + print(results) + ``` + + 1. Creates a new, empty audience. The `add_compare_example` calls train and filter the labelers who join it. + + !!! note + Review every qualification example and its truth carefully, and add more than the few shown here for production workloads — see [Custom Audiences](../audiences.md) for the full guide. diff --git a/docs/examples/locate_job.md b/docs/examples/locate_job.md new file mode 100644 index 000000000..26472983e --- /dev/null +++ b/docs/examples/locate_job.md @@ -0,0 +1,108 @@ +# Locate Job Example + +To learn about the basics of creating a job, please refer to the [quickstart guide](../quickstart.md). + +In a locate job, labelers tap the points in a datapoint that match your instruction. In this example, we ask people to point out visual artifacts in AI-generated images — a common way to find where a generator went wrong. + +Like any other job, a locate job can be assigned to any audience — a ready-to-go curated one, or a custom audience you train with qualification examples. + +=== "Simple" + + The simple version runs straight away on a **curated** audience — a pre-existing pool of labelers, ready to work immediately — so the job starts collecting responses right away. + + ```python + from rapidata import RapidataClient + + IMAGE_URLS = [ + "https://assets.rapidata.ai/eac11c3e-ad57-402b-90ed-23378d2ff869.jpg", + "https://assets.rapidata.ai/04e7e3c6-5554-47ca-bdb2-950e48ac3e6c.jpg", + "https://assets.rapidata.ai/91d9913c-b399-47f8-ad19-767798cc951c.jpg", + ] + + client = RapidataClient() + + audience = client.audience.get_audience_by_id("global") # (1)! + + job_definition = client.job.create_locate_job_definition( + name="Artifact Detection Example", + instruction="Tap on any visual glitches or errors in the image.", # (2)! + datapoints=IMAGE_URLS, + responses_per_datapoint=35, + ) + + job_definition.preview() + + job = audience.assign_job(job_definition) + job.display_progress_bar() + results = job.get_results() + print(results) + ``` + + 1. The global audience (id `global`) already has labelers ready to work, so the job starts collecting responses immediately. You can assign a locate job to any audience — browse them in the [Rapidata Dashboard](https://app.rapidata.ai/audiences). + 2. The instruction tells labelers what to locate. Each response is the set of points they tapped on that datapoint. + +=== "Advanced" + + The advanced version builds a **custom** audience and trains labelers with qualification examples before running the job. Each example carries the bounding box(es) covering the region a correct labeler should tap; only labelers who tap inside them join the audience, which raises label quality. + + !!! warning "This takes significantly longer" + Unlike the Simple path, this first builds and trains an entirely new audience before the job can start collecting responses — expect it to take considerably longer to return results. + + ```python + from rapidata import RapidataClient, Box + + IMAGE_URLS = [ + "https://assets.rapidata.ai/eac11c3e-ad57-402b-90ed-23378d2ff869.jpg", + "https://assets.rapidata.ai/04e7e3c6-5554-47ca-bdb2-950e48ac3e6c.jpg", + "https://assets.rapidata.ai/91d9913c-b399-47f8-ad19-767798cc951c.jpg", + ] + + # Qualification examples — each pairs an image with the bounding box(es) + # covering the region a correct labeler should tap. Coordinates are image + # ratios (0.0–1.0); + EXAMPLES = [ + ("https://assets.rapidata.ai/544b1210-1e91-4351-a97c-fe8263b319b4.webp", + [Box(x_min=0.44, y_min=0.42, x_max=0.58, y_max=0.63)]), + ("https://assets.rapidata.ai/f1e11611-7c5b-4186-8ddf-51e06c0859ff.webp", + [Box(x_min=0.07, y_min=0.37, x_max=0.39, y_max=0.71)]), + ("https://assets.rapidata.ai/ad816f8f-f7a9-4c90-90dd-9c10bc556856.webp", + [Box(x_min=0.04, y_min=0.10, x_max=0.31, y_max=0.28)]), + ("https://assets.rapidata.ai/a076ae24-4d5c-415d-9d41-6afbe2fbfcde.webp", + [Box(x_min=0.25, y_min=0.40, x_max=0.70, y_max=0.96)]), + ("https://assets.rapidata.ai/38753cb4-4b77-4fb7-b601-8a5bc3d166d7.webp", + [Box(x_min=0.41, y_min=0.09, x_max=0.87, y_max=0.45)]), + ("https://assets.rapidata.ai/50109592-b521-4dcb-a00f-453f6c026a52.webp", + [Box(x_min=0.25, y_min=0.03, x_max=0.71, y_max=0.48)]), + ("https://assets.rapidata.ai/a5a954d0-91e8-4b4e-bec6-2bb739444be8.webp", + [Box(x_min=0.57, y_min=0.40, x_max=0.96, y_max=0.89)]), + ] + + client = RapidataClient() + + audience = client.audience.create_audience(name="Artifact Detection Audience") # (1)! + for datapoint, truths in EXAMPLES: + audience.add_locate_example( + instruction="Tap on any visual glitches or errors in the image.", + datapoint=datapoint, + truths=truths, + ) + + job_definition = client.job.create_locate_job_definition( + name="Artifact Detection Example", + instruction="Tap on any visual glitches or errors in the image.", + datapoints=IMAGE_URLS, + responses_per_datapoint=35, + ) + + job_definition.preview() + + job = audience.assign_job(job_definition) + job.display_progress_bar() + results = job.get_results() + print(results) + ``` + + 1. Creates a new, empty audience. The `add_locate_example` calls train and filter the labelers who join it. + + !!! note + Review every qualification example and its truth regions carefully, and add more than the few shown here for production workloads — see [Custom Audiences](../audiences.md) for the full guide. diff --git a/docs/job_definition_parameters.md b/docs/job_definition_parameters.md index 188c8b6e7..b0b5697f2 100644 --- a/docs/job_definition_parameters.md +++ b/docs/job_definition_parameters.md @@ -67,6 +67,7 @@ The data to be labeled. The format depends on the job type: |----------|--------|-------------| | Classification | `list[str]` | Single items to classify | | Compare | `list[list[str]]` | Pairs of items (exactly 2 per inner list) | +| Locate | `list[str]` | Single items to locate within | **Supported Formats:** @@ -337,21 +338,33 @@ job_definition = client.job.create_compare_job_definition( ) ``` +### Locate Job + +Locate has no job-specific parameters — it uses only the core parameters. The `instruction` describes what labelers should locate, and each response is the set of points they tapped on the datapoint. + +```python +job_definition = client.job.create_locate_job_definition( + name="Artifact Detection", + instruction="Tap on any visual glitches or errors in the image.", + datapoints=["image1.jpg", "image2.jpg"], +) +``` + --- ## Parameter Availability Matrix -| Parameter | Classification | Compare | -|-----------|:-:|:-:| -| `name` | :white_check_mark: | :white_check_mark: | -| `instruction` | :white_check_mark: | :white_check_mark: | -| `datapoints` | :white_check_mark: | :white_check_mark: | -| `responses_per_datapoint` | :white_check_mark: | :white_check_mark: | -| `data_type` | :white_check_mark: | :white_check_mark: | -| `contexts` | :white_check_mark: | :white_check_mark: | -| `media_contexts` | :white_check_mark: | :white_check_mark: | -| `confidence_threshold` | :white_check_mark: | :white_check_mark: | -| `quorum_threshold` | :white_check_mark: | :white_check_mark: | -| `settings` | :white_check_mark: | :white_check_mark: | -| `answer_options` | :white_check_mark: | :x: | -| `a_b_names` | :x: | :white_check_mark: | +| Parameter | Classification | Compare | Locate | +|-----------|:-:|:-:|:-:| +| `name` | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| `instruction` | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| `datapoints` | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| `responses_per_datapoint` | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| `data_type` | :white_check_mark: | :white_check_mark: | :x: | +| `contexts` | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| `media_contexts` | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| `confidence_threshold` | :white_check_mark: | :white_check_mark: | :x: | +| `quorum_threshold` | :white_check_mark: | :white_check_mark: | :x: | +| `settings` | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| `answer_options` | :white_check_mark: | :x: | :x: | +| `a_b_names` | :x: | :white_check_mark: | :x: | diff --git a/docs/quickstart.md b/docs/quickstart.md index 9c256f918..4f40bbdab 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -61,10 +61,10 @@ client = RapidataClient(client_id="Your client ID", client_secret="Your client s The simplest way to get started is with a curated audience: ```py -audience = client.audience.find_audiences("alignment")[0] # (1)! +audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO") # (1)! ``` -1. Curated audiences are pre-existing pools of labelers trained on a specific type of task. +1. Curated audiences are pre-existing pools of labelers trained on a specific type of task — this is the **Alignment** audience. You can browse the curated audiences and copy their ids from the [Rapidata Dashboard](https://app.rapidata.ai/audiences). !!! note The curated audience gets you started quickly, but results may be less accurate than a custom audience trained with examples specific to your task. For higher quality, see [Custom Audiences](audiences.md). @@ -165,7 +165,7 @@ from rapidata import RapidataClient client = RapidataClient() -audience = client.audience.find_audiences("alignment")[0] +audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO") job_definition = client.job.create_compare_job_definition( name="Example Image Prompt Alignment", diff --git a/docs/starting_page.md b/docs/starting_page.md index 63b902496..e2e6b1c6e 100644 --- a/docs/starting_page.md +++ b/docs/starting_page.md @@ -18,7 +18,7 @@ The SDK has three building blocks: **audiences** (who labels), **job definitions client = RapidataClient() - audience = client.audience.find_audiences("alignment")[0] + audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO") job_definition = client.job.create_compare_job_definition( name="Example Image Comparison", @@ -41,7 +41,7 @@ The SDK has three building blocks: **audiences** (who labels), **job definitions client = RapidataClient() - audience = client.audience.find_audiences("alignment")[0] + audience = client.audience.get_audience_by_id("aud_MU1GZYoESyO") job_definition = client.job.create_compare_job_definition( name="Example Video Comparison", @@ -149,5 +149,6 @@ The SDK is built around three concepts: |---|---|---| | **Compare** | Side-by-side comparison of images, video, audio, or text | [Comparison example](examples/compare_job.md) | | **Classify** | Categorize data with custom labels or Likert scales | [Classification example](examples/classify_job.md) | +| **Locate** | Point out objects, artifacts, or regions within an image | [Locate example](examples/locate_job.md) | | **Rank models** | Benchmark AI models on leaderboards with human evaluation | [Model Ranking](mri.md) | | **Continuous ranking** | Lightweight ongoing ranking without full job setup | [Ranking Flows](flows.md) | diff --git a/mkdocs.yml b/mkdocs.yml index f46934382..793dc2714 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -68,6 +68,7 @@ plugins: Examples: - examples/classify_job.md - examples/compare_job.md + - examples/locate_job.md Benchmarks: - mri.md - mri_advanced.md @@ -124,6 +125,7 @@ nav: - Examples: - Classification: examples/classify_job.md - Comparison: examples/compare_job.md + - Locate: examples/locate_job.md - Ranking Flows: flows.md - Model Ranking: - Getting Started: mri.md diff --git a/src/rapidata/rapidata_client/audience/audience_example_handler.py b/src/rapidata/rapidata_client/audience/audience_example_handler.py index ded438c66..faa761768 100644 --- a/src/rapidata/rapidata_client/audience/audience_example_handler.py +++ b/src/rapidata/rapidata_client/audience/audience_example_handler.py @@ -19,6 +19,16 @@ from rapidata.api_client.models.i_example_truth_compare_example_truth import ( IExampleTruthCompareExampleTruth, ) +from rapidata.api_client.models.i_example_payload_locate_example_payload import ( + IExamplePayloadLocateExamplePayload, +) +from rapidata.api_client.models.i_example_truth_locate_example_truth import ( + IExampleTruthLocateExampleTruth, +) +from rapidata.rapidata_client.validation.rapids.box import ( + Box, + calculate_boxes_coverage, +) from rapidata.service.openapi_service import OpenAPIService from rapidata.api_client.models.i_example_payload import IExamplePayload from rapidata.api_client.models.i_example_truth import IExampleTruth @@ -186,6 +196,66 @@ def add_compare_example( ), ) + def add_locate_example( + self, + instruction: str, + datapoint: str, + truths: list[Box], + context: str | None = None, + media_context: list[str] | None = None, + explanation: str | None = None, + settings: Sequence[RapidataSetting] | None = None, + ) -> None: + """add a locate example to the audience + + Args: + instruction (str): The instruction telling the labeler what to locate. + datapoint (str): The media datapoint the labeler will be locating the target in. + truths (list[Box]): The bounding boxes covering the correct regions to tap. Coordinates are ratios of the image size (0.0 to 1.0). + context (str, optional): The context is text that will be shown in addition to the instruction. Defaults to None. + media_context (list[str], optional): A list of image URLs / paths that will be shown in addition to the instruction (can be combined with context). Pass a single-element list for one image, or multiple to display several images. Defaults to None. + explanation (str, optional): The explanation that will be shown to the labeler if the answer is wrong. Defaults to None. + settings (Sequence[RapidataSetting], optional): The list of settings to apply to the example as feature flags. Controls how the example is rendered to the labeler. Defaults to None. + """ + from rapidata.api_client.models.add_example_to_audience_endpoint_input import ( + AddExampleToAudienceEndpointInput, + ) + + if not truths: + raise ValueError("Locate example requires at least one truth bounding box") + + asset_input = self._asset_uploader.upload_and_map_asset(datapoint) + + payload = IExamplePayload( + actual_instance=IExamplePayloadLocateExamplePayload( + _t="LocateExamplePayload", target=instruction + ) + ) + model_truth = IExampleTruth( + actual_instance=IExampleTruthLocateExampleTruth( + _t="LocateExampleTruth", + boundingBoxes=[truth.to_example_model() for truth in truths], + ) + ) + + self._openapi_service.audience.examples_api.audience_audience_id_example_post( + audience_id=self._audience_id, + add_example_to_audience_endpoint_input=AddExampleToAudienceEndpointInput( + asset=asset_input, + payload=payload, + truth=model_truth, + context=context, + contextAsset=( + self._asset_uploader.upload_and_map_asset(media_context) + if media_context + else None + ), + explanation=explanation, + randomCorrectProbability=calculate_boxes_coverage(truths), + featureFlags=[s._to_feature_flag() for s in settings] if settings else None, + ), + ) + def _add_rapid_example(self, rapid: Rapid) -> None: """Add a rapid example to the audience (private method). diff --git a/src/rapidata/rapidata_client/audience/rapidata_audience.py b/src/rapidata/rapidata_client/audience/rapidata_audience.py index 3929d4006..a45c1437a 100644 --- a/src/rapidata/rapidata_client/audience/rapidata_audience.py +++ b/src/rapidata/rapidata_client/audience/rapidata_audience.py @@ -15,6 +15,7 @@ ) from rapidata.rapidata_client.filter import RapidataFilter from rapidata.rapidata_client.validation.rapids.rapids import Rapid + from rapidata.rapidata_client.validation.rapids.box import Box from rapidata.rapidata_client.settings._rapidata_setting import RapidataSetting import pandas as pd @@ -266,6 +267,50 @@ def add_compare_example( self._try_start_recruiting() return self + def add_locate_example( + self, + instruction: str, + datapoint: str, + truths: list[Box], + context: str | None = None, + media_context: list[str] | None = None, + explanation: str | None = None, + settings: Sequence[RapidataSetting] | None = None, + ) -> RapidataAudience: + """Add a locate training example to this audience. + + Training examples help annotators understand the task by showing them + a sample datapoint with the correct regions before they start labeling. + + Args: + instruction (str): The instruction telling annotators what to locate. + datapoint (str): The media datapoint (URL or path) to use as the training example. + truths (list[Box]): The bounding boxes covering the correct regions to tap, as :class:`Box` objects with coordinates in image ratios (0.0 to 1.0). + context (str, optional): Additional text context to display with the example. Defaults to None. + media_context (list[str], optional): Additional image URLs / paths to display with the example. Pass a single-element list for one image, or multiple to display several images. Defaults to None. + explanation (str, optional): An explanation of why the truth is correct. Defaults to None. + settings (Sequence[RapidataSetting], optional): Settings applied as feature flags on this example so the qualification example matches how the actual task will be rendered. Defaults to None. + + Returns: + RapidataAudience: The audience instance (self) for method chaining. + """ + media_context = coerce_media_context(media_context) + with tracer.start_as_current_span("RapidataAudience.add_locate_example"): + logger.debug( + f"Adding locate example to audience: {self.id} with instruction: {instruction}, datapoint: {datapoint}, truths: {truths}, context: {context}, media_context: {media_context}, explanation: {explanation}, settings: {settings}" + ) + self._example_handler.add_locate_example( + instruction, + datapoint, + truths, + context, + media_context, + explanation, + settings, + ) + self._try_start_recruiting() + return self + def get_examples( self, amount: int = 10, diff --git a/src/rapidata/rapidata_client/job/rapidata_job_manager.py b/src/rapidata/rapidata_client/job/rapidata_job_manager.py index 4bc01ae4f..e83172de2 100644 --- a/src/rapidata/rapidata_client/job/rapidata_job_manager.py +++ b/src/rapidata/rapidata_client/job/rapidata_job_manager.py @@ -498,7 +498,7 @@ def _create_select_words_job_definition( settings=settings, ) - def _create_locate_job_definition( + def create_locate_job_definition( self, name: str, instruction: str, diff --git a/src/rapidata/rapidata_client/validation/rapids/box.py b/src/rapidata/rapidata_client/validation/rapids/box.py index 9b6b9041a..73ab80754 100644 --- a/src/rapidata/rapidata_client/validation/rapids/box.py +++ b/src/rapidata/rapidata_client/validation/rapids/box.py @@ -1,6 +1,7 @@ from rapidata.api_client.models.locate_box_truth_model_box import ( LocateBoxTruthModelBox, ) +from rapidata.api_client.models.example_box_shape import ExampleBoxShape from pydantic import BaseModel, field_validator, model_validator @@ -42,3 +43,60 @@ def to_model(self) -> LocateBoxTruthModelBox: xMax=self.x_max * 100, yMax=self.y_max * 100, ) + + def to_example_model(self) -> ExampleBoxShape: + return ExampleBoxShape( + xMin=self.x_min * 100, + yMin=self.y_min * 100, + xMax=self.x_max * 100, + yMax=self.y_max * 100, + ) + + +def calculate_boxes_coverage(boxes: list[Box]) -> float: + """Calculate the ratio of image area covered by a list of boxes. + + Args: + boxes: List of Box objects with coordinates in range [0, 1]. + + Returns: + float: Coverage ratio between 0.0 and 1.0. + """ + if not boxes: + return 0.0 + + # Sweep line over x: at each x-interval, sum the merged y-coverage of the + # currently active boxes, weighted by the interval width. + events: list[tuple[float, str, int]] = [] + for i, box in enumerate(boxes): + events.append((box.x_min, "start", i)) + events.append((box.x_max, "end", i)) + + events.sort(key=lambda e: (e[0], e[1] == "end")) + + total_area = 0.0 + active_boxes: set[int] = set() + prev_x = 0.0 + + for x, event_type, box_id in events: + if active_boxes and x > prev_x: + y_intervals = sorted( + (boxes[i].y_min, boxes[i].y_max) for i in active_boxes + ) + merged: list[tuple[float, float]] = [] + for start, end in y_intervals: + if merged and start <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + else: + merged.append((start, end)) + y_coverage = sum(end - start for start, end in merged) + total_area += (x - prev_x) * y_coverage + + if event_type == "start": + active_boxes.add(box_id) + else: + active_boxes.discard(box_id) + + prev_x = x + + return total_area diff --git a/src/rapidata/rapidata_client/validation/rapids/rapids_manager.py b/src/rapidata/rapidata_client/validation/rapids/rapids_manager.py index b01c2c396..617b30f59 100644 --- a/src/rapidata/rapidata_client/validation/rapids/rapids_manager.py +++ b/src/rapidata/rapidata_client/validation/rapids/rapids_manager.py @@ -6,7 +6,10 @@ from rapidata.rapidata_client.validation.rapids.rapids import Rapid from rapidata.service.openapi_service import OpenAPIService -from rapidata.rapidata_client.validation.rapids.box import Box +from rapidata.rapidata_client.validation.rapids.box import ( + Box, + calculate_boxes_coverage, +) class RapidsManager: @@ -431,57 +434,7 @@ def _calculate_boxes_coverage(self, boxes: list[Box]) -> float: Returns: float: Coverage ratio between 0.0 and 1.0 """ - if not boxes: - return 0.0 - - # Convert boxes to intervals for sweep line algorithm - events = [] - - # Create events for x-coordinates - for i, box in enumerate(boxes): - events.append((box.x_min, "start", i, box)) - events.append((box.x_max, "end", i, box)) - - # Sort events by x-coordinate - events.sort(key=lambda x: (x[0], x[1] == "end")) - - total_area = 0.0 - active_boxes = set() - prev_x = 0.0 - - for x, event_type, box_id, box in events: - # Calculate area for the previous x-interval - if active_boxes and x > prev_x: - # Merge y-intervals for active boxes - y_intervals = [(boxes[i].y_min, boxes[i].y_max) for i in active_boxes] - y_intervals.sort() - - # Merge overlapping y-intervals - merged_intervals = [] - for start, end in y_intervals: - if merged_intervals and start <= merged_intervals[-1][1]: - # Overlapping intervals - merge them - merged_intervals[-1] = ( - merged_intervals[-1][0], - max(merged_intervals[-1][1], end), - ) - else: - # Non-overlapping interval - merged_intervals.append((start, end)) - - # Calculate total y-coverage for this x-interval - y_coverage = sum(end - start for start, end in merged_intervals) - total_area += (x - prev_x) * y_coverage - - # Update active boxes - if event_type == "start": - active_boxes.add(box_id) - else: - active_boxes.discard(box_id) - - prev_x = x - - return total_area + return calculate_boxes_coverage(boxes) @staticmethod def _calculate_coverage_ratio(