feat(timeseries): add time-series analysis extension with embedding similarity and drift detection

timeseries/CMakeLists.txt

@@ -0,0 +1,9 @@
+include_directories(
+        ${PROJECT_SOURCE_DIR}/src/include
+        ${CMAKE_BINARY_DIR}/src/include
+        src/include)
+
+add_subdirectory(src/main)
+add_subdirectory(src/function)
+
+build_extension_lib(${BUILD_STATIC_EXTENSION} "timeseries")

timeseries/README.md

@@ -0,0 +1,70 @@
+# LadybugDB Timeseries Extension
+
+Time-series analysis functions for sequential data: embedding similarity comparison and drift detection on embedding sequences.
+
+## Functions
+
+### EMBEDDING_SIMILARITY
+
+Compute cosine similarity and per-dimension feature similarity between two embedding vectors of any dimension.
+
+```cypher
+CALL embedding_similarity(
+    CAST([0.1, 0.2, 0.3], 'DOUBLE[]'),
+    CAST([0.1, 0.2, 0.4], 'DOUBLE[]')
+) RETURN cosine_similarity, feature_similarity, dimension_count;
+```
+
+**Parameters:**
+- `vec_a LIST<DOUBLE>` — first embedding vector
+- `vec_b LIST<DOUBLE>` — second embedding vector
+
+**Returns:**
+- `cosine_similarity DOUBLE` — cosine similarity [-1, 1]
+- `feature_similarity DOUBLE` — mean per-dimension feature similarity [0, 1]
+- `dimension_count INT64` — number of dimensions compared
+
+### DETECT_DRIFT_POINTS
+
+Detect drift points in a sequence of embeddings by computing pairwise cosine distances between consecutive vectors. Returns drift points sorted by significance.
+
+```cypher
+CALL detect_drift_points(
+    CAST([1.0, 0.0, 0.0, 0.0, 1.0, 0.0], 'DOUBLE[]'),
+    2,                    -- num_embeddings
+    CAST([100, 200], 'INT64[]'),  -- labels
+    3,                    -- embedding_dim
+    threshold := 0.1
+) RETURN label, drift_magnitude, significance, direction;
+```
+
+**Parameters:**
+- `flat_embeddings LIST<DOUBLE>` — N×D doubles (row-major: e1_1, e1_2, ..., eN_D)
+- `num_embeddings INT64` — N
+- `labels LIST<INT64>` — N labels (commit hashes, timestamps, version numbers, etc.)
+- `embedding_dim INT64` — D (e.g. 768)
+- `threshold:=0.3` — optional DOUBLE detection threshold (default 0.3)
+
+**Returns:**
+- `label INT64` — label of the drift point
+- `drift_magnitude DOUBLE` — raw cosine distance
+- `significance DOUBLE` — normalized significance [0, 1]
+- `direction STRING` — "up" or "down" relative to previous distance
+
+## Use Cases
+
+- **Architecture drift**: embeddings of code snapshots per commit → detect when architecture changed significantly
+- **Content drift**: embeddings of document revisions over time → detect when content drifted
+- **Behavior drift**: embeddings of API response patterns → detect behavioral changes
+- **Vector comparison**: compare any two embeddings regardless of dimension
+
+## Building
+
+```bash
+cmake -DBUILD_EXTENSIONS="timeseries" ..
+cmake --build . --target libtimeseries.lbug_extension
+```
+
+## Dependencies
+
+No external dependencies. Pure C++ implementation.

timeseries/src/function/CMakeLists.txt

@@ -0,0 +1,9 @@
+add_library(lbug_timeseries_function
+        OBJECT
+        embedding_similarity.cpp
+        detect_drift_points.cpp
+        )
+
+set(TIMESERIES_EXTENSION_OBJECT_FILES
+        ${TIMESERIES_EXTENSION_OBJECT_FILES} $<TARGET_OBJECTS:lbug_timeseries_function>
+        PARENT_SCOPE)

timeseries/src/function/detect_drift_points.cpp

@@ -0,0 +1,171 @@
+// DETECT_DRIFT_POINTS — generic drift detection on sequential embeddings
+//
+// CALL detect_drift_points(
+//     flat_embeddings LIST<DOUBLE>,   -- N×D doubles (row-major: e1_1, e1_2, ..., eN_D)
+//     num_embeddings INT64,            -- N
+//     labels LIST<INT64>,              -- N labels (e.g. chapter numbers, commit hashes, timestamps)
+//     embedding_dim INT64,             -- D (e.g. 768)
+//     threshold:=0.3                   -- [optional] DOUBLE detection threshold
+// ) RETURN
+//     label INT64, drift_magnitude DOUBLE,
+//     significance DOUBLE, direction STRING
+//
+// Algorithm:
+//   1. Reconstruct N vectors of D dimensions from flat list
+//   2. Compute pairwise cosine distance between consecutive vectors
+//   3. Min-max normalize distances → significance [0,1]
+//   4. Filter by threshold, sort by significance descending
+//
+// Use cases:
+//   - Architecture drift: embeddings of code snapshots per commit
+//   - Content drift: embeddings of document revisions over time
+//   - Behavior drift: embeddings of API response patterns
+//
+// Pure computation — no table scans. Data prepared via Cypher COLLECT at app layer.
+
+#include "binder/binder.h"
+#include "binder/expression/literal_expression.h"
+#include "common/types/value/nested.h"
+#include "common/types/value/value.h"
+#include "function/table/bind_data.h"
+#include "function/table/bind_input.h"
+#include "function/table/simple_table_function.h"
+#include "function/timeseries_function.h"
+#include "main/client_context.h"
+#include "processor/execution_context.h"
+#include <algorithm>
+#include <cmath>
+#include <string>
+#include <vector>
+
+using namespace lbug::binder;
+using namespace lbug::common;
+using namespace lbug::function;
+using namespace lbug::processor;
+
+namespace lbug { namespace timeseries_extension {
+
+struct DDPBD final : TableFuncBindData {
+    std::vector<double> embeds;     // N×D doubles (row-major)
+    std::vector<int64_t> labels;    // N labels
+    int64_t dims;                   // D
+    int64_t numEmb;                 // N
+    double thr;                     // detection threshold
+    DDPBD(std::vector<double> e, std::vector<int64_t> l, int64_t d, int64_t n, double t,
+          expression_vector co, row_idx_t nr)
+        : TableFuncBindData{std::move(co),nr}, embeds{std::move(e)}, labels{std::move(l)},
+          dims{d}, numEmb{n}, thr{t} {}
+    std::unique_ptr<TableFuncBindData> copy() const override {
+        return std::make_unique<DDPBD>(embeds,labels,dims,numEmb,thr,columns,numRows);
+    }
+};
+
+// Cosine distance: 1 − dot(a,b)/(|a|×|b|), range [0,2]
+static double cosDist(const double* a, const double* b, int64_t dims) {
+    double d = 0, nA = 0, nB = 0;
+    for (int64_t i = 0; i < dims; i++) { d += a[i]*b[i]; nA += a[i]*a[i]; nB += b[i]*b[i]; }
+    if (nA < 1e-12 || nB < 1e-12) return 0.0;
+    double cs = d / std::sqrt(nA * nB);
+    if (cs > 1.0) cs = 1.0; if (cs < -1.0) cs = -1.0;
+    return 1.0 - cs;
+}
+
+static offset_t tableFunc(const TableFuncMorsel&, const TableFuncInput& in, DataChunk& out) {
+    auto bd = in.bindData->constPtrCast<DDPBD>();
+    int64_t N = bd->numEmb, D = bd->dims;
+    if (N < 2) return 0;
+
+    std::vector<double> dist(N - 1);
+    for (int64_t i = 1; i < N; i++)
+        dist[i-1] = cosDist(&bd->embeds[(i-1)*D], &bd->embeds[i*D], D);
+
+    double mn = *std::min_element(dist.begin(), dist.end());
+    double mx = *std::max_element(dist.begin(), dist.end());
+    double rng = mx - mn + 0.001;
+
+    struct DP { int64_t label; double mag, sig; std::string dir; };
+    std::vector<DP> dps;
+    for (int64_t i = 0; i < N-1; i++) {
+        if (dist[i] > bd->thr) {
+            double sig = (dist[i] - mn) / rng;
+            std::string dir = (i > 0 && dist[i] > dist[i-1]) ? "up" : "down";
+            dps.push_back({bd->labels[i+1], dist[i], sig, dir});
+        }
+    }
+
+    std::sort(dps.begin(), dps.end(),
+              [](const DP& a, const DP& b) { return a.sig > b.sig; });
+
+    for (size_t j = 0; j < dps.size(); j++) {
+        out.getValueVectorMutable(0).setValue((offset_t)j, dps[j].label);
+        out.getValueVectorMutable(1).setValue((offset_t)j, dps[j].mag);
+        out.getValueVectorMutable(2).setValue((offset_t)j, dps[j].sig);
+        out.getValueVectorMutable(3).setValue((offset_t)j, dps[j].dir);
+    }
+    return (offset_t)dps.size();
+}
+
+static std::unique_ptr<TableFuncBindData> bindFunc(const main::ClientContext*,
+    const TableFuncBindInput* in) {
+    // flat_embeddings (param 0) — LIST<DOUBLE>
+    auto fv = in->getValue(0);
+    uint32_t nf = NestedVal::getChildrenSize(&fv);
+    std::vector<double> embeds; embeds.reserve(nf);
+    for (uint32_t j = 0; j < nf; j++)
+        embeds.push_back(NestedVal::getChildVal(&fv, j)->getValue<double>());
+
+    // num_embeddings (param 1) — INT64
+    int64_t numEmb = in->getValue(1).getValue<int64_t>();
+
+    // labels (param 2) — LIST<INT64>
+    auto cv = in->getValue(2);
+    uint32_t nc = NestedVal::getChildrenSize(&cv);
+    std::vector<int64_t> labels; labels.reserve(nc);
+    for (uint32_t j = 0; j < nc; j++)
+        labels.push_back(NestedVal::getChildVal(&cv, j)->getValue<int64_t>());
+
+    // embedding_dim (param 3) — INT64
+    int64_t dims = in->getValue(3).getValue<int64_t>();
+
+    // threshold (optional param, default 0.3)
+    double thr = 0.3;
+    for (auto& p : in->optionalParamsLegacy)
+        if (p->getAlias() == "threshold")
+            if (auto le = p->constPtrCast<LiteralExpression>())
+                thr = le->getValue().getValue<double>();
+
+    if (numEmb <= 0 || dims <= 0 || (int64_t)nf != numEmb * dims || (int64_t)nc != numEmb)
+        numEmb = 0;
+
+    std::vector<std::string> ns = {"label","drift_magnitude","significance","direction"};
+    std::vector<LogicalType> ts; ts.reserve(4);
+    ts.push_back(LogicalType::INT64()); ts.push_back(LogicalType::DOUBLE());
+    ts.push_back(LogicalType::DOUBLE()); ts.push_back(LogicalType::STRING());
+    ns = TableFunction::extractYieldVariables(ns, in->yieldVariables);
+    row_idx_t maxRows = (row_idx_t)(numEmb > 1 ? numEmb - 1 : 1);
+    return std::make_unique<DDPBD>(std::move(embeds), std::move(labels), dims, numEmb, thr,
+                                   in->binder->createVariables(ns, ts), maxRows);
+}
+
+function_set DetectDriftPointsFunction::getFunctionSet() {
+    function_set fs;
+    auto f = std::make_unique<TableFunction>(name,
+        std::vector{LogicalTypeID::ANY, LogicalTypeID::INT64,
+                    LogicalTypeID::ANY, LogicalTypeID::INT64});
+    f->inferInputTypes = [](const expression_vector&) -> std::vector<LogicalType> {
+        std::vector<LogicalType> result; result.reserve(4);
+        result.push_back(LogicalType::LIST(LogicalType::DOUBLE()));
+        result.push_back(LogicalType::INT64());
+        result.push_back(LogicalType::LIST(LogicalType::INT64()));
+        result.push_back(LogicalType::INT64());
+        return result;
+    };
+    f->tableFunc = SimpleTableFunc::getTableFunc(tableFunc);
+    f->bindFunc = bindFunc;
+    f->initSharedStateFunc = SimpleTableFunc::initSharedState;
+    f->initLocalStateFunc = TableFunction::initEmptyLocalState;
+    fs.push_back(std::move(f));
+    return fs;
+}
+
+}} // namespaces

timeseries/src/function/embedding_similarity.cpp

@@ -0,0 +1,129 @@
+// EMBEDDING_SIMILARITY — generic N-dimension feature similarity
+//
+// CALL embedding_similarity(
+//     vec_a LIST<DOUBLE>,  -- first embedding
+//     vec_b LIST<DOUBLE>,  -- second embedding
+// ) RETURN similarity_score DOUBLE, dimension_count INT64
+//
+// Algorithm:
+//   1. Compute cosine similarity: dot(a,b) / (|a| × |b|)
+//   2. Compute per-dimension feature similarity: 1 - |a-b| / max(|a|,|b|,eps)
+//   3. Return cosine similarity as primary score + dimension count
+//
+// Generalized from bitemporal's character_similarity (which was hardcoded to 4 dimensions).
+// Works with embeddings of any dimension (768-dim text, 384-dim code, etc.)
+
+#include "binder/binder.h"
+#include "function/table/bind_data.h"
+#include "function/table/bind_input.h"
+#include "function/table/simple_table_function.h"
+#include "function/timeseries_function.h"
+#include "main/client_context.h"
+#include "processor/execution_context.h"
+#include <cmath>
+#include <string>
+#include <vector>
+
+using namespace lbug::binder;
+using namespace lbug::common;
+using namespace lbug::function;
+using namespace lbug::processor;
+
+namespace lbug { namespace timeseries_extension {
+
+struct ESBD final : TableFuncBindData {
+    std::vector<double> vecA;
+    std::vector<double> vecB;
+    ESBD(std::vector<double> a, std::vector<double> b, expression_vector c, row_idx_t n)
+        : TableFuncBindData{std::move(c),n}, vecA{std::move(a)}, vecB{std::move(b)} {}
+    std::unique_ptr<TableFuncBindData> copy() const override {
+        return std::make_unique<ESBD>(vecA,vecB,columns,numRows);
+    }
+};
+
+// Cosine similarity: dot(a,b) / (|a| × |b|)
+static double cosineSim(const std::vector<double>& a, const std::vector<double>& b) {
+    double dot = 0, nA = 0, nB = 0;
+    size_t len = std::min(a.size(), b.size());
+    for (size_t i = 0; i < len; i++) {
+        dot += a[i] * b[i];
+        nA += a[i] * a[i];
+        nB += b[i] * b[i];
+    }
+    if (nA < 1e-12 || nB < 1e-12) return 0.0;
+    double cs = dot / (std::sqrt(nA) * std::sqrt(nB));
+    if (cs > 1.0) cs = 1.0;
+    if (cs < -1.0) cs = -1.0;
+    return cs;
+}
+
+// Mean per-dimension feature similarity
+static double meanFeatureSim(const std::vector<double>& a, const std::vector<double>& b) {
+    size_t len = std::min(a.size(), b.size());
+    if (len == 0) return 0.0;
+    double sum = 0;
+    for (size_t i = 0; i < len; i++) {
+        double d = std::max(std::max(std::abs(a[i]), std::abs(b[i])), 0.001);
+        sum += 1.0 - std::abs(a[i] - b[i]) / d;
+    }
+    return sum / len;
+}
+
+static offset_t tableFunc(const TableFuncMorsel&, const TableFuncInput& in, DataChunk& out) {
+    auto bd = in.bindData->constPtrCast<ESBD>();
+    double cosSim = cosineSim(bd->vecA, bd->vecB);
+    double featSim = meanFeatureSim(bd->vecA, bd->vecB);
+    int64_t dims = (int64_t)std::min(bd->vecA.size(), bd->vecB.size());
+
+    auto pos = out.state->getSelVector()[0];
+    out.getValueVectorMutable(0).setValue(pos, cosSim);
+    out.getValueVectorMutable(1).setValue(pos, featSim);
+    out.getValueVectorMutable(2).setValue(pos, dims);
+    return 1;
+}
+
+static std::unique_ptr<TableFuncBindData> bindFunc(const main::ClientContext*,
+    const TableFuncBindInput* in) {
+    // vec_a (param 0) — LIST<DOUBLE>
+    auto av = in->getValue(0);
+    uint32_t na = NestedVal::getChildrenSize(&av);
+    std::vector<double> vecA; vecA.reserve(na);
+    for (uint32_t j = 0; j < na; j++)
+        vecA.push_back(NestedVal::getChildVal(&av, j)->getValue<double>());
+
+    // vec_b (param 1) — LIST<DOUBLE>
+    auto bv = in->getValue(1);
+    uint32_t nb = NestedVal::getChildrenSize(&bv);
+    std::vector<double> vecB; vecB.reserve(nb);
+    for (uint32_t j = 0; j < nb; j++)
+        vecB.push_back(NestedVal::getChildVal(&bv, j)->getValue<double>());
+
+    std::vector<std::string> ns = {"cosine_similarity", "feature_similarity", "dimension_count"};
+    std::vector<LogicalType> ts;
+    ts.push_back(LogicalType::DOUBLE());
+    ts.push_back(LogicalType::DOUBLE());
+    ts.push_back(LogicalType::INT64());
+    ns = TableFunction::extractYieldVariables(ns, in->yieldVariables);
+    return std::make_unique<ESBD>(std::move(vecA), std::move(vecB),
+                                  in->binder->createVariables(ns, ts), 1);
+}
+
+function_set EmbeddingSimilarityFunction::getFunctionSet() {
+    function_set fs;
+    auto f = std::make_unique<TableFunction>(name,
+        std::vector{LogicalTypeID::ANY, LogicalTypeID::ANY});
+    f->inferInputTypes = [](const expression_vector&) -> std::vector<LogicalType> {
+        std::vector<LogicalType> result;
+        result.push_back(LogicalType::LIST(LogicalType::DOUBLE()));
+        result.push_back(LogicalType::LIST(LogicalType::DOUBLE()));
+        return result;
+    };
+    f->tableFunc = SimpleTableFunc::getTableFunc(tableFunc);
+    f->bindFunc = bindFunc;
+    f->initSharedStateFunc = SimpleTableFunc::initSharedState;
+    f->initLocalStateFunc = TableFunction::initEmptyLocalState;
+    fs.push_back(std::move(f));
+    return fs;
+}
+
+}} // namespaces