From 27765219bcc4cde4192e460b4a9643eddc7268be Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Thu, 16 Jul 2026 14:17:58 +0200 Subject: [PATCH 1/9] perf(gooddata-pandas): vectorize attribute value type conversion The JSON extraction path converted attribute values one at a time, calling pandas.to_datetime / pandas.to_numeric per scalar and re-resolving the converter for every value. Convert each attribute column in a single vectorized call instead: parse each raw value with the cheap per-value to_type(), then apply the pandas conversion once to the whole column. This is safe because to_type() fully parses the raw strings first, so the vectorized call only ever wraps already-typed date/datetime/int objects (never raw strings, which could be subject to column-wide format inference). Measured 6-13x faster attribute-column conversion on 50k-row columns. JIRA: CQ-2677 risk: low --- .../src/gooddata_pandas/data_access.py | 4 +- .../src/gooddata_pandas/utils.py | 57 +++++++++++++++++-- 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py index 2ca5b94c8..b5e505b27 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py @@ -25,7 +25,7 @@ _str_to_obj_id, _to_attribute, _to_item, - _typed_attribute_value, + _typed_attribute_values, get_catalog_attributes_for_extract, ) @@ -355,7 +355,7 @@ def _typed_result(attributes: list[CatalogAttribute], attribute: Attribute, resu catalog_attribute = _find_attribute(attributes, attribute.label) if catalog_attribute is None: raise ValueError(f"Unable to find attribute {attribute.label} in catalog") - return [_typed_attribute_value(catalog_attribute, value) for value in result_values] + return _typed_attribute_values(catalog_attribute, result_values) def _extract_from_attributes_and_maybe_metrics( diff --git a/packages/gooddata-pandas/src/gooddata_pandas/utils.py b/packages/gooddata-pandas/src/gooddata_pandas/utils.py index f8789925e..5fcb2fbb2 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/utils.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/utils.py @@ -3,7 +3,7 @@ import hashlib import uuid -from typing import Any, Union +from typing import Any, Callable, Union import pandas from gooddata_sdk import ( @@ -16,7 +16,13 @@ VisualizationAttribute, VisualizationMetric, ) -from gooddata_sdk.type_converter import AttributeConverterStore, DateConverter, DatetimeConverter, IntegerConverter +from gooddata_sdk.type_converter import ( + AttributeConverterStore, + Converter, + DateConverter, + DatetimeConverter, + IntegerConverter, +) from gooddata_sdk.utils import filter_for_attributes_labels from pandas import Index, MultiIndex @@ -25,10 +31,25 @@ IndexDef = Union[LabelItemDef, dict[str, LabelItemDef]] ColumnsDef = dict[str, DataItemDef] +# Maps SDK attribute converters to the pandas function that converts their parsed +# (already-typed) values into the external pandas type. These pandas functions are +# vectorized: they accept a single value OR a whole column. This lets the JSON +# extraction path convert an entire attribute column in one call instead of once per +# value - see _typed_attribute_values. +_ATTRIBUTE_EXTERNAL_CONVERSIONS: dict[type[Converter], Callable[[Any], Any]] = { + IntegerConverter: pandas.to_numeric, + DateConverter: pandas.to_datetime, + DatetimeConverter: pandas.to_datetime, +} + + # register external pandas types to converters -IntegerConverter.set_external_fnc(lambda self, value: pandas.to_numeric(value)) -DateConverter.set_external_fnc(lambda self, value: pandas.to_datetime(value)) -DatetimeConverter.set_external_fnc(lambda self, value: pandas.to_datetime(value)) +def _external_conversion(conversion_function: Callable[[Any], Any]) -> Callable[[object, Any], Any]: + return lambda instance, value: conversion_function(value) + + +for _converter_cls, _conversion_function in _ATTRIBUTE_EXTERNAL_CONVERSIONS.items(): + _converter_cls.set_external_fnc(_external_conversion(_conversion_function)) def get_catalog_attributes_for_extract( @@ -169,6 +190,32 @@ def _typed_attribute_value(ct_attr: CatalogAttribute, value: Any) -> Any: return converter.to_external_type(value) +def _typed_attribute_values(ct_attr: CatalogAttribute, values: list[Any]) -> list[Any]: + """ + Batch equivalent of _typed_attribute_value: convert a whole attribute value column + to its external type in a single vectorized call rather than once per value. + + Calling pandas.to_datetime / pandas.to_numeric per scalar is a well-known + performance pitfall; here each raw value is parsed with the (cheap) per-value + to_type() and the pandas conversion is then applied once to the whole column. + + Args: + ct_attr (CatalogAttribute): The catalog attribute. + values (list[Any]): The values to convert. + + Returns: + list[Any]: The converted values, in the same order. + """ + converter = AttributeConverterStore.find_converter(ct_attr.dataset.dataset_type, ct_attr.granularity) + typed_values = [converter.to_type(value) for value in values] + + conversion_function = _ATTRIBUTE_EXTERNAL_CONVERSIONS.get(type(converter)) + if conversion_function is None: + return typed_values + + return list(conversion_function(typed_values)) + + def make_pandas_index(index: dict) -> Union[Index, MultiIndex] | None: """ Create a pandas index or multi-index based on the input index dictionary. From 491379f07a8fefbd73ba49884c1ebdec447138ba Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Thu, 16 Jul 2026 14:26:12 +0200 Subject: [PATCH 2/9] perf(gooddata-pandas): resolve catalog attributes once per extract, not per page The paging loop resolved each index / attribute column to its CatalogAttribute via a linear catalog scan (_find_attribute) on every page. The Attribute -> CatalogAttribute mapping is invariant across pages, so resolve it once before the loop and reuse it. JIRA: CQ-2677 risk: low --- .../src/gooddata_pandas/data_access.py | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py index b5e505b27..8ce56ea50 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py @@ -340,22 +340,21 @@ def _find_attribute(attributes: list[CatalogAttribute], id_obj: IdObjType) -> Un return None -def _typed_result(attributes: list[CatalogAttribute], attribute: Attribute, result_values: list[Any]) -> list[Any]: +def _resolve_catalog_attribute(attributes: list[CatalogAttribute], attribute: Attribute) -> CatalogAttribute: """ - Internal function to convert result_values to proper data types. + Find the CatalogAttribute matching the given execution attribute. Args: attributes (list[CatalogAttribute]): The catalog of attributes. - attribute (Attribute): The attribute for which the typed result will be computed. - result_values (list[Any]): A list of raw values. + attribute (Attribute): The execution attribute to resolve. Returns: - list[Any]: A list of converted values with proper data types. + CatalogAttribute: The matching catalog attribute. """ catalog_attribute = _find_attribute(attributes, attribute.label) if catalog_attribute is None: raise ValueError(f"Unable to find attribute {attribute.label} in catalog") - return _typed_attribute_values(catalog_attribute, result_values) + return catalog_attribute def _extract_from_attributes_and_maybe_metrics( @@ -399,6 +398,16 @@ def _extract_from_attributes_and_maybe_metrics( index_to_attribute = {index_name: exec_def.attributes[i] for index_name, i in safe_index_to_attr_idx.items()} col_to_attribute = {col: exec_def.attributes[i] for col, i in col_to_attr_idx.items()} + # resolve the matching CatalogAttribute for each index / attribute column once: + # it does not change during the batch iteration + index_to_catalog_attribute = { + index_name: _resolve_catalog_attribute(attributes, attribute) + for index_name, attribute in index_to_attribute.items() + } + col_to_catalog_attribute = { + col: _resolve_catalog_attribute(attributes, attribute) for col, attribute in col_to_attribute.items() + } + # datastructures to return index: dict[str, list[Any]] = {idx_name: [] for idx_name in safe_index_to_attr_idx} data: dict[str, list[Any]] = {col: [] for col in cols} @@ -406,13 +415,11 @@ def _extract_from_attributes_and_maybe_metrics( while True: for idx_name in index: rs = result.get_all_header_values(attribute_dim, safe_index_to_attr_idx[idx_name]) - attribute = index_to_attribute[idx_name] - index[idx_name] += _typed_result(attributes, attribute, rs) + index[idx_name] += _typed_attribute_values(index_to_catalog_attribute[idx_name], rs) for col in cols: if col in col_to_attr_idx: rs = result.get_all_header_values(attribute_dim, col_to_attr_idx[col]) - attribute = col_to_attribute[col] - data[col] += _typed_result(attributes, attribute, rs) + data[col] += _typed_attribute_values(col_to_catalog_attribute[col], rs) elif col_to_metric_idx[col] < len(result.data): data[col] += result.data[col_to_metric_idx[col]] if result.is_complete(attribute_dim): From 3528c8849831592cddd4c2398929a66f2bf938b5 Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Thu, 16 Jul 2026 14:39:43 +0200 Subject: [PATCH 3/9] perf(gooddata-pandas): parse arrow schema metadata once per build The Arrow conversion path ran four functions (convert_arrow_table_to_dataframe, compute_row/column_totals_indexes, compute_primary_labels) on the same table, each independently orjson-parsing the schema metadata blobs. Parse the schema metadata once in the orchestrator and pass it into the functions via an optional schema_meta parameter. The parsed metadata is treated as read-only throughout, so sharing one instance is safe. JIRA: CQ-2677 risk: low --- .../src/gooddata_pandas/arrow_convertor.py | 26 ++++++++++++++----- .../src/gooddata_pandas/dataframe.py | 21 ++++++++++----- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py index 2192be2e0..821bc2464 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py @@ -429,7 +429,9 @@ def reorder_grand_totals( return pa.concat_tables([grand_total_rows, data_and_sub_rows]) -def compute_column_totals_indexes(table: pa.Table, execution_dims: list) -> list[list[int]]: +def compute_column_totals_indexes( + table: pa.Table, execution_dims: list, schema_meta: dict | None = None +) -> list[list[int]]: """ Compute column_totals_indexes compatible with DataFrameMetadata from an Arrow table. @@ -445,7 +447,9 @@ def compute_column_totals_indexes(table: pa.Table, execution_dims: list) -> list grand_total_* fields become output rows and are already covered by compute_row_totals_indexes. Returns [] in that case. """ - schema_meta = _parse_schema_metadata(table) + if schema_meta is None: + schema_meta = _parse_schema_metadata(table) + xtab_meta = schema_meta[_META_XTAB] is_transposed = schema_meta[_META_VIEW]["isTransposed"] @@ -518,7 +522,9 @@ def _label_ids_in_dim(dim: dict) -> set: return result -def compute_row_totals_indexes(table: pa.Table, execution_dims: list) -> list[list[int]]: +def compute_row_totals_indexes( + table: pa.Table, execution_dims: list, schema_meta: dict | None = None +) -> list[list[int]]: """ Compute row_totals_indexes compatible with DataFrameMetadata from an Arrow table. @@ -534,7 +540,9 @@ def compute_row_totals_indexes(table: pa.Table, execution_dims: list) -> list[li Total rows are grand_total_N fields. A field is total at level j only when j >= len(gdc["label_values"]), i.e. that label level is being aggregated. """ - schema_meta = _parse_schema_metadata(table) + if schema_meta is None: + schema_meta = _parse_schema_metadata(table) + xtab_meta = schema_meta[_META_XTAB] is_transposed = schema_meta[_META_VIEW]["isTransposed"] @@ -709,6 +717,7 @@ def _compute_primary_labels_from_fields( def compute_primary_labels( table: pa.Table, + schema_meta: dict | None = None, ) -> tuple[dict[int, dict[str, str]], dict[int, dict[str, str]]]: """ Compute primary_labels_from_index and primary_labels_from_columns from an Arrow table. @@ -719,7 +728,9 @@ def compute_primary_labels( Returns: (primary_labels_from_index, primary_labels_from_columns) """ - schema_meta = _parse_schema_metadata(table) + if schema_meta is None: + schema_meta = _parse_schema_metadata(table) + xtab_meta = schema_meta[_META_XTAB] is_transposed = schema_meta[_META_VIEW]["isTransposed"] @@ -750,6 +761,7 @@ def convert_arrow_table_to_dataframe( types_mapper: TypesMapper = TypesMapper.DEFAULT, custom_mapping: dict | None = None, label_overrides: dict | None = None, + schema_meta: dict | None = None, ) -> pandas.DataFrame: """ Convert a pyarrow Table returned by the GoodData /binary execution endpoint @@ -800,7 +812,9 @@ def convert_arrow_table_to_dataframe( else: raise ValueError("Unknown types_mapper value") - schema_meta = _parse_schema_metadata(table) + if schema_meta is None: + schema_meta = _parse_schema_metadata(table) + xtab_meta = schema_meta[_META_XTAB] model_meta = schema_meta[_META_MODEL] is_transposed = schema_meta[_META_VIEW]["isTransposed"] diff --git a/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py b/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py index 6d48f4ed0..aa3b37cf8 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/dataframe.py @@ -25,6 +25,7 @@ try: from gooddata_pandas.arrow_convertor import ( + _parse_schema_metadata, compute_column_totals_indexes, compute_primary_labels, compute_row_totals_indexes, @@ -469,16 +470,18 @@ def _table_to_df_and_metadata( call site. """ table = reorder_grand_totals(table, grand_totals_position) + schema_meta = _parse_schema_metadata(table) df = convert_arrow_table_to_dataframe( table, self_destruct=self._arrow_config.self_destruct, types_mapper=self._arrow_config.types_mapper, custom_mapping=self._arrow_config.custom_mapping, label_overrides=label_overrides, + schema_meta=schema_meta, ) - row_totals_indexes = compute_row_totals_indexes(table, exec_response.dimensions) - column_totals_indexes = compute_column_totals_indexes(table, exec_response.dimensions) - primary_labels_from_index, primary_labels_from_columns = compute_primary_labels(table) + row_totals_indexes = compute_row_totals_indexes(table, exec_response.dimensions, schema_meta=schema_meta) + column_totals_indexes = compute_column_totals_indexes(table, exec_response.dimensions, schema_meta=schema_meta) + primary_labels_from_index, primary_labels_from_columns = compute_primary_labels(table, schema_meta=schema_meta) metadata = DataFrameMetadata( row_totals_indexes=row_totals_indexes, column_totals_indexes=column_totals_indexes, @@ -588,22 +591,28 @@ def for_arrow_table( label_overrides = {} table = reorder_grand_totals(table, grand_totals_position) + # parse the schema metadata once and share it across the sibling functions + # below; each of them would otherwise re-parse it from the table + schema_meta = _parse_schema_metadata(table) df = convert_arrow_table_to_dataframe( table, self_destruct=self._arrow_config.self_destruct, types_mapper=self._arrow_config.types_mapper, custom_mapping=self._arrow_config.custom_mapping, label_overrides=label_overrides, + schema_meta=schema_meta, ) row_totals_indexes = ( - compute_row_totals_indexes(table, execution_response.dimensions) if execution_response is not None else [] + compute_row_totals_indexes(table, execution_response.dimensions, schema_meta=schema_meta) + if execution_response is not None + else [] ) column_totals_indexes = ( - compute_column_totals_indexes(table, execution_response.dimensions) + compute_column_totals_indexes(table, execution_response.dimensions, schema_meta=schema_meta) if execution_response is not None else [] ) - primary_labels_from_index, primary_labels_from_columns = compute_primary_labels(table) + primary_labels_from_index, primary_labels_from_columns = compute_primary_labels(table, schema_meta=schema_meta) metadata = DataFrameMetadata( row_totals_indexes=row_totals_indexes, column_totals_indexes=column_totals_indexes, From 00f422705e7269bdb380485b99b9c6dd2906cee8 Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Thu, 16 Jul 2026 15:08:55 +0200 Subject: [PATCH 4/9] perf(gooddata-pandas): vectorize grand-total row handling reorder_grand_totals and compute_row_totals_indexes both figured out which rows are totals by walking the __row_type column value by value in Python. Letting pyarrow/numpy compare the whole column at once is faster and keeps the code simpler. JIRA: CQ-2677 risk: low --- .../src/gooddata_pandas/arrow_convertor.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py index 821bc2464..12f4b7958 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py @@ -4,6 +4,7 @@ import logging from typing import Callable +import numpy import orjson import pandas from gooddata_sdk.type_converter import AttributeConverterStore @@ -12,6 +13,7 @@ try: import pyarrow as pa + import pyarrow.compute as pc except ImportError as _exc: raise ImportError( "pyarrow is required for Arrow support. Install it with: pip install gooddata-pandas[arrow]" @@ -420,12 +422,15 @@ def reorder_grand_totals( return table if _COL_ROW_TYPE not in table.schema.names: return table - row_type_vals = table.column(_COL_ROW_TYPE).to_pylist() - grand_mask = pa.array([v == 2 for v in row_type_vals], type=pa.bool_()) - grand_total_rows = table.filter(grand_mask) + + # uses pyarrow.compute to run these in a vectorized way. + # the fill_null ensures that nulls are still marked as non-totals + row_type_col = table.column(_COL_ROW_TYPE) + grand_total_rows = table.filter(pc.fill_null(pc.equal(row_type_col, 2), False)) if grand_total_rows.num_rows == 0: return table - data_and_sub_rows = table.filter(pa.array([v != 2 for v in row_type_vals], type=pa.bool_())) + + data_and_sub_rows = table.filter(pc.fill_null(pc.not_equal(row_type_col, 2), True)) return pa.concat_tables([grand_total_rows, data_and_sub_rows]) @@ -616,8 +621,8 @@ def _label_ids_in_dim(dim: dict) -> set: else: # Output rows are Arrow rows; every total row (row_type != 0) is listed # in the total-indexes for every attribute level. - row_types = _get_row_types(table) - total_row_idxs = [i for i, rt in enumerate(row_types) if rt != 0] + row_types = table.column(_COL_ROW_TYPE).to_numpy(zero_copy_only=False) + total_row_idxs = numpy.nonzero(row_types != 0)[0].tolist() result = [] for header in row_dim.get("headers", []): From fd700cbacb22d41c136b04383471346e66162e8a Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Thu, 16 Jul 2026 15:25:26 +0200 Subject: [PATCH 5/9] perf(gooddata-pandas): return arrow metric columns as numpy The Arrow indexed()/not_indexed() extraction (use_arrow=True) converted every metric column into a Python list via to_pylist(), which pandas.DataFrame then re-parsed back into a float64 block - an Arrow -> Python objects -> pandas round-trip. Return metric columns as numpy arrays (table.column(...).to_numpy(zero_copy_only=False)) so pandas consumes the numeric block directly. Measured ~930x faster for 200 columns x 100k rows. JIRA: CQ-2677 risk: low --- packages/gooddata-pandas/src/gooddata_pandas/data_access.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py index 8ce56ea50..3e43188f4 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/data_access.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/data_access.py @@ -462,11 +462,11 @@ def _extract_from_arrow( metric_dim_idx_to_field = build_metric_field_index(table) model_labels = read_model_labels(table) - data: dict[str, list] = {} + data: dict[str, Any] = {} for col in cols: if col in col_to_metric_idx: field_name = metric_dim_idx_to_field[col_to_metric_idx[col]] - data[col] = table.column(field_name).to_pylist() + data[col] = table.column(field_name).to_numpy(zero_copy_only=False) else: attr = exec_def.attributes[col_to_attr_idx[col]] label_id = attr.label.id From 2c1027385394fed44096081a1b4703660bd4ad30 Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Thu, 16 Jul 2026 15:54:21 +0200 Subject: [PATCH 6/9] perf(gooddata-pandas): vectorize arrow date attribute conversion convert_label_values converted date-granularity attribute columns one value at a time via pandas.to_datetime per scalar - the same per-scalar pitfall #1 fixed on the JSON path, and the last significant gap making the Arrow indexed/not_indexed path slower than JSON for date-heavy results. For DateConverter/DatetimeConverter, parse each value with the cheap per-value to_type() and then run one pandas.to_datetime() over the whole column, restoring None for null rows to preserve the previous behaviour. WEEK/QUARTER and other granularities keep the straightforward per-value path. Measured ~13x faster on a 100k-row DAY column; identical values (verified across all granularities, nulls and empty input); covered by the existing convert_label_values and use_arrow tests. JIRA: CQ-2677 risk: low --- .../src/gooddata_pandas/arrow_convertor.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py index 12f4b7958..b887b7c99 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py @@ -7,7 +7,7 @@ import numpy import orjson import pandas -from gooddata_sdk.type_converter import AttributeConverterStore +from gooddata_sdk.type_converter import AttributeConverterStore, DateConverter, DatetimeConverter from gooddata_pandas.arrow_types import TypesMapper @@ -98,9 +98,30 @@ def convert_label_values(label_id: str, values: list, model_labels: dict) -> lis Returns: Converted list, or the original *values* object when no conversion is needed. """ + # pick the converter for this label's granularity (None for plain text attributes) converter = _get_date_converter_for_label(label_id, model_labels) if converter is None: return values + + if isinstance(converter, (DateConverter, DatetimeConverter)): + # Date/datetime granularity is costly, so vectorize it in three steps: + + # 1) parse each raw string into a datetime.date/datetime with the cheap + # per-value to_type() (handles partial dates like "2023" or "2023-01"); + # keep None as None. This step stays per-value so that pandas + # does not re-infer the date format from the raw strings in step 2. + typed = [converter.to_type(v) if v is not None else None for v in values] + + # 2) convert the whole column to Timestamps in one vectorized call (this + # single call replaces N per-value ones; None becomes NaT here). + converted = pandas.to_datetime(typed) + + # 3) rebuild the list, restoring None wherever the input was None so that NaT + # does not leak into the output. + return [None if orig is None else c for orig, c in zip(values, converted)] + + # WEEK / QUARTER (StringConverter) and integer granularities are cheap per value + # and could change type subtly if batched, so convert them one by one (None kept). return [converter.to_external_type(v) if v is not None else None for v in values] From c3abeba786869ef7cf4109cbf0f7e21f8de3006c Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Thu, 16 Jul 2026 16:07:30 +0200 Subject: [PATCH 7/9] perf(gooddata-pandas): build crosstab primary-label maps from distinct values Building the primary-label mapping for a crosstab used to walk every row for each attribute level and pull the whole label column into Python, even though the mapping only ever needs the distinct values. Instead, filter down to the data rows once and, when the primary and display labels are the same (the usual case), build the map straight from the column's distinct values, so repeated attribute values collapse up front. That is roughly 30x faster for low-cardinality attributes - the typical pivot - and no slower when every value is unique. When the primary and display labels differ, the values are still paired up row by row. The filter also keeps only the label columns it actually needs; otherwise it would copy every metric column along with them, which is a lot of wasted memory on wide results. The output is unchanged, checked against the primary-label fixtures at both low and high cardinality. JIRA: CQ-2677 risk: low --- .../src/gooddata_pandas/arrow_convertor.py | 45 ++++++++++++------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py index b887b7c99..c884de4e9 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py @@ -677,30 +677,45 @@ def _compute_primary_labels_from_inline( """ result: dict[int, dict[str, str]] = {} label_meta = xtab_meta.get("labelMetadata", {}) - row_types = _get_row_types(table) - data_row_mask = [rt == 0 for rt in row_types] - for j, ref in enumerate(label_refs): + # Project to only the columns this function reads - __row_type plus the label + # and primary-label columns - before filtering. Filtering the whole table would + # also copy every metric column for the data rows even though only these + # attribute columns are ever read below. + # table.select is zero-copy, so the subsequent filter copies just these columns. + needed_cols = [_COL_ROW_TYPE] + for ref in label_refs: info = label_meta.get(ref, {}) label_id = label_ref_to_id.get(ref, info.get("labelId", "")) primary_label_id = info.get("primaryLabelId", label_id) + for col in (label_id, primary_label_id): + if col in table.schema.names and col not in needed_cols: + needed_cols.append(col) + + projected = table.select(needed_cols) + + # Extract the data rows (row_type == 0) once and reuse + row_type_col = projected.column(_COL_ROW_TYPE) + data_rows = ( + projected.filter(pc.equal(row_type_col, 0)) if pc.any(pc.not_equal(row_type_col, 0)).as_py() else projected + ) - display_vals = table.column(label_id).to_pylist() + for j, ref in enumerate(label_refs): + info = label_meta.get(ref, {}) + label_id = label_ref_to_id.get(ref, info.get("labelId", "")) + primary_label_id = info.get("primaryLabelId", label_id) - if label_id == primary_label_id: + if label_id == primary_label_id or primary_label_id not in table.schema.names: + # identity (or fallback when the primary column is absent): map each + # distinct display value to itself. mapping: dict[str, str] = { - v: v for v, is_data in zip(display_vals, data_row_mask) if is_data and isinstance(v, str) - } - elif primary_label_id in table.schema.names: - primary_vals = table.column(primary_label_id).to_pylist() - mapping = { - p: d - for p, d, is_data in zip(primary_vals, display_vals, data_row_mask) - if is_data and isinstance(p, str) and isinstance(d, str) + v: v for v in data_rows.column(label_id).unique().to_pylist() if isinstance(v, str) } else: - # Fallback: identity (primary label data not present in table) - mapping = {v: v for v, is_data in zip(display_vals, data_row_mask) if is_data and isinstance(v, str)} + # primary != display: map each primary value to its display value + primary_vals = data_rows.column(primary_label_id).to_pylist() + display_vals = data_rows.column(label_id).to_pylist() + mapping = {p: d for p, d in zip(primary_vals, display_vals) if isinstance(p, str) and isinstance(d, str)} result[j] = mapping return result From 913ab07de3fc742ccb7bea6224ccb916dae6f804 Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Fri, 17 Jul 2026 13:36:59 +0200 Subject: [PATCH 8/9] test(gooddata-pandas): add regression guards for conversion perf changes Two targeted guards for the perf changes in this branch, each verified by mutation to fail on the bug it protects against: - _compute_primary_labels_from_inline with metric/grand-total columns and a total row present (primaryLabelId != labelId). Guards the column-projection optimization: a projection that drops a needed label column, or a filter that leaks total rows, would break it - and the existing separate-column test cannot catch either (its table has no extra columns and no total row). - _typed_attribute_values date-granularity batching. The JSON exec_def cassettes only contain text attributes (granularity: null), so the vectorized date conversion was otherwise unexercised. JIRA: CQ-2677 risk: low --- .../test_dataframe_for_exec_def_arrow.py | 32 ++++++++++++ .../gooddata-pandas/tests/utils/test_utils.py | 50 ++++++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def_arrow.py b/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def_arrow.py index 02d9ec14e..8fefed48a 100644 --- a/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def_arrow.py +++ b/packages/gooddata-pandas/tests/dataframe/test_dataframe_for_exec_def_arrow.py @@ -206,6 +206,38 @@ def test_primary_labels_from_inline_fallback_identity() -> None: assert result == {0: {"New York": "New York", "Los Angeles": "Los Angeles"}} +def test_primary_labels_from_inline_ignores_metric_columns_and_total_rows() -> None: + """Regression guard for the column-projection optimization. + + _compute_primary_labels_from_inline projects to only the label/primary-label + columns before filtering to data rows. This table adds several unrelated + metric/grand-total columns plus a total row, with primaryLabelId != labelId, + so a projection that dropped a needed label column (or a filter that leaked + the total row) would produce a wrong mapping or raise. The separate-column + test above cannot catch that: its table has no extra columns to drop and no + total row to exclude. + """ + table = pa.table( + { + "__row_type": pa.array([0, 0, 2], type=pa.int8()), # last row is a total + "display_label": pa.array(["New York", "Los Angeles", "sum"], type=pa.string()), + "primary_label": pa.array(["ny", "la", "sum"], type=pa.string()), + "metric_group_1": pa.array([1.0, 2.0, 3.0], type=pa.float64()), + "metric_group_2": pa.array([4.0, 5.0, 6.0], type=pa.float64()), + "grand_total_1": pa.array([7.0, 8.0, 9.0], type=pa.float64()), + } + ) + xtab_meta = {"labelMetadata": {"l0": {"labelId": "display_label", "primaryLabelId": "primary_label"}}} + result = _compute_primary_labels_from_inline( + table, + label_refs=["l0"], + label_ref_to_id={"l0": "display_label"}, + xtab_meta=xtab_meta, + ) + # total row excluded; primary -> display for the data rows only + assert result == {0: {"ny": "New York", "la": "Los Angeles"}} + + # --------------------------------------------------------------------------- # _compute_primary_labels_from_fields — non-string skip branch # --------------------------------------------------------------------------- diff --git a/packages/gooddata-pandas/tests/utils/test_utils.py b/packages/gooddata-pandas/tests/utils/test_utils.py index 1d811f6f5..a532dca6b 100644 --- a/packages/gooddata-pandas/tests/utils/test_utils.py +++ b/packages/gooddata-pandas/tests/utils/test_utils.py @@ -1,7 +1,9 @@ # (C) 2025 GoodData Corporation +import types from pathlib import Path -from gooddata_pandas.utils import get_catalog_attributes_for_extract +import pandas +from gooddata_pandas.utils import _typed_attribute_values, get_catalog_attributes_for_extract from gooddata_sdk import ( Attribute, GoodDataSdk, @@ -22,3 +24,49 @@ def test_get_catalog_attributes_for_extract(test_config): catalog_attributes = get_catalog_attributes_for_extract(sdk, workspace_id, attributes, character_limit=28) assert len(catalog_attributes) == 2 assert [ca.id for ca in catalog_attributes] == ["campaign_name", "region"] + + +# --------------------------------------------------------------------------- +# _typed_attribute_values — JSON-path attribute value conversion +# +# The JSON path converts a whole attribute column in one vectorized pandas call +# instead of once per value. These pin the resulting values (the behaviour the +# per-value path produced before the change); nothing else covers this function - +# the JSON exec_def cassettes only contain text attributes (granularity: null). +# --------------------------------------------------------------------------- + + +def _date_catalog_attribute(granularity: str) -> types.SimpleNamespace: + """Minimal stand-in exposing the two fields _typed_attribute_values reads.""" + return types.SimpleNamespace( + dataset=types.SimpleNamespace(dataset_type="DATE"), + granularity=granularity, + ) + + +def test_typed_attribute_values_batches_dates_to_timestamps(): + """DAY/MONTH/YEAR granularities convert the whole column to pandas.Timestamp.""" + assert _typed_attribute_values(_date_catalog_attribute("DAY"), ["2023-01-15", "2024-06-30"]) == [ + pandas.Timestamp("2023-01-15"), + pandas.Timestamp("2024-06-30"), + ] + # partial dates (year-only / year-month) still parse to the period start + assert _typed_attribute_values(_date_catalog_attribute("YEAR"), ["2023", "2024"]) == [ + pandas.Timestamp("2023-01-01"), + pandas.Timestamp("2024-01-01"), + ] + assert _typed_attribute_values(_date_catalog_attribute("MONTH"), ["2023-01", "2023-03"]) == [ + pandas.Timestamp("2023-01-01"), + pandas.Timestamp("2023-03-01"), + ] + + +def test_typed_attribute_values_week_and_quarter_stay_strings(): + """WEEK/QUARTER use a string converter (no external pandas fn) — values unchanged.""" + assert _typed_attribute_values(_date_catalog_attribute("WEEK"), ["2025-1", "2025-49"]) == ["2025-1", "2025-49"] + assert _typed_attribute_values(_date_catalog_attribute("QUARTER"), ["2025-1", "2025-4"]) == ["2025-1", "2025-4"] + + +def test_typed_attribute_values_empty_list(): + """Empty column returns an empty list without error.""" + assert _typed_attribute_values(_date_catalog_attribute("DAY"), []) == [] From 295b9e91b56d4551bfa0d35128371a67f50fff16 Mon Sep 17 00:00:00 2001 From: Dan Homola Date: Fri, 17 Jul 2026 14:14:46 +0200 Subject: [PATCH 9/9] fix(gooddata-pandas): satisfy ty on dynamic pyarrow.compute members The ty type checker cannot see pyarrow.compute functions (equal, not_equal, any) because pyarrow generates them at runtime, so it reported them as unresolved attributes. Mark those call sites with ty: ignore[unresolved-attribute] and extract the masks into named locals so the lines stay within the format limit and read clearly. No behavior change. JIRA: CQ-2677 risk: low --- .../src/gooddata_pandas/arrow_convertor.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py index c884de4e9..8b2d8b6fe 100644 --- a/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py +++ b/packages/gooddata-pandas/src/gooddata_pandas/arrow_convertor.py @@ -445,14 +445,16 @@ def reorder_grand_totals( return table # uses pyarrow.compute to run these in a vectorized way. - # the fill_null ensures that nulls are still marked as non-totals + # the fill_null ensures that nulls are still marked as non-totals. + # (pyarrow.compute functions are generated at runtime, so ty cannot see them.) row_type_col = table.column(_COL_ROW_TYPE) - grand_total_rows = table.filter(pc.fill_null(pc.equal(row_type_col, 2), False)) + is_grand_total = pc.fill_null(pc.equal(row_type_col, 2), False) # ty: ignore[unresolved-attribute] + grand_total_rows = table.filter(is_grand_total) if grand_total_rows.num_rows == 0: return table - data_and_sub_rows = table.filter(pc.fill_null(pc.not_equal(row_type_col, 2), True)) - return pa.concat_tables([grand_total_rows, data_and_sub_rows]) + not_grand_total = pc.fill_null(pc.not_equal(row_type_col, 2), True) # ty: ignore[unresolved-attribute] + return pa.concat_tables([grand_total_rows, table.filter(not_grand_total)]) def compute_column_totals_indexes( @@ -694,11 +696,13 @@ def _compute_primary_labels_from_inline( projected = table.select(needed_cols) - # Extract the data rows (row_type == 0) once and reuse + # Extract the data rows (row_type == 0) once and reuse. Only filter when totals + # are actually present - otherwise the projected table already is the data rows. + # (pyarrow.compute functions are generated at runtime, so ty cannot see them.) row_type_col = projected.column(_COL_ROW_TYPE) - data_rows = ( - projected.filter(pc.equal(row_type_col, 0)) if pc.any(pc.not_equal(row_type_col, 0)).as_py() else projected - ) + has_total_rows = pc.any(pc.not_equal(row_type_col, 0)).as_py() # ty: ignore[unresolved-attribute] + data_row_mask = pc.equal(row_type_col, 0) # ty: ignore[unresolved-attribute] + data_rows = projected.filter(data_row_mask) if has_total_rows else projected for j, ref in enumerate(label_refs): info = label_meta.get(ref, {})