diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 5473aa8812e..bf3e65d8741 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -36,8 +36,6 @@ public enum Key { PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), PPL_SUBSEARCH_MAXOUT("plugins.ppl.subsearch.maxout"), PPL_JOIN_SUBSEARCH_MAXOUT("plugins.ppl.join.subsearch_maxout"), - PPL_REST_REDACTION_ENABLED("plugins.ppl.rest.redaction.enabled"), - PPL_REST_ALLOWED_ENDPOINTS("plugins.ppl.rest.allowed_endpoints"), /** Enable Calcite as execution engine */ CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"), diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java b/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java deleted file mode 100644 index d0a0e34d3c9..00000000000 --- a/core/src/main/java/org/opensearch/sql/ast/tree/RestRelation.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ast.tree; - -import java.util.Collections; -import lombok.EqualsAndHashCode; -import lombok.ToString; -import org.opensearch.sql.ast.expression.UnresolvedExpression; - -/** - * Extend Relation to mark a {@code rest} leading command. The single table name is a reserved, - * encoded token (produced by {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}) that - * carries the validated REST endpoint spec; it resolves through the storage engine to a REST source - * table on the Calcite path, exactly as {@link DescribeRelation} resolves to a system index. - */ -@ToString -@EqualsAndHashCode(callSuper = false) -public class RestRelation extends Relation { - public RestRelation(UnresolvedExpression tableName) { - super(Collections.singletonList(tableName)); - } -} diff --git a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java index 7589cf522f6..c9fa35d7068 100644 --- a/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/SystemIndexUtils.java @@ -5,9 +5,6 @@ package org.opensearch.sql.utils; -import java.nio.charset.StandardCharsets; -import java.util.LinkedHashMap; -import java.util.Map; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.experimental.UtilityClass; @@ -37,121 +34,6 @@ public static Boolean isSystemIndex(String indexName) { return indexName.endsWith(SYS_TABLES_SUFFIX); } - /** - * Reserved suffix marking a {@code rest} source table. Distinct from {@link #SYS_TABLES_SUFFIX} - * so {@link #isSystemIndex} and {@link #isRestSource} never overlap. The whole reserved name is a - * single Calcite identifier (REST + lowercase hex + this suffix), so it survives name resolution - * the same way the uppercase system-mapping names do. - */ - private static final String REST_SOURCE_SUFFIX = "__REST_SOURCE"; - - private static final String REST_SOURCE_PREFIX = "REST"; - - /** True if the resolved table name is a {@code rest} source token. */ - public static boolean isRestSource(String indexName) { - return indexName.endsWith(REST_SOURCE_SUFFIX); - } - - /** - * Encode a validated {@link RestSpec} into a single reserved table name. Mirrors {@link - * #mappingTable}: structured metadata travels inside a reserved name rather than a side channel. - * The endpoint/args have already been allow-list-validated before this is called. - */ - public static String restTable(RestSpec spec) { - StringBuilder sb = new StringBuilder(); - sb.append("endpoint=").append(spec.getEndpoint()); - if (spec.getCount() != null) { - sb.append('\n').append("count=").append(spec.getCount()); - } - if (spec.getTimeout() != null) { - sb.append('\n').append("timeout=").append(spec.getTimeout()); - } - if (spec.getArgs() != null) { - for (Map.Entry e : spec.getArgs().entrySet()) { - sb.append('\n').append("arg.").append(e.getKey()).append('=').append(e.getValue()); - } - } - return REST_SOURCE_PREFIX + toHex(sb.toString()) + REST_SOURCE_SUFFIX; - } - - /** Decode a reserved {@code rest} table name back into its {@link RestSpec}. */ - public static RestSpec decodeRestSpec(String indexName) { - // Validate the token shape before slicing it: callers gate on isRestSource today, but a - // public decoder must not assume its precondition, otherwise a malformed token would throw an - // opaque StringIndexOutOfBoundsException from substring rather than a clear input error. - if (!isRestSource(indexName)) { - throw new IllegalArgumentException("not a valid rest source token: " + indexName); - } - String body = - indexName.substring( - REST_SOURCE_PREFIX.length(), indexName.length() - REST_SOURCE_SUFFIX.length()); - String decoded = fromHex(body); - String endpoint = null; - Integer count = null; - String timeout = null; - LinkedHashMap args = new LinkedHashMap<>(); - for (String line : decoded.split("\n")) { - if (line.isEmpty()) { - continue; - } - int eq = line.indexOf('='); - if (eq < 0) { - continue; - } - String k = line.substring(0, eq); - String v = line.substring(eq + 1); - if (k.equals("endpoint")) { - endpoint = v; - } else if (k.equals("count")) { - count = Integer.parseInt(v); - } else if (k.equals("timeout")) { - timeout = v; - } else if (k.startsWith("arg.")) { - args.put(k.substring("arg.".length()), v); - } - } - if (endpoint == null) { - throw new IllegalArgumentException("rest source token is missing the endpoint: " + indexName); - } - return new RestSpec(endpoint, args, count, timeout); - } - - private static String toHex(String s) { - StringBuilder h = new StringBuilder(); - for (byte b : s.getBytes(StandardCharsets.UTF_8)) { - h.append(Character.forDigit((b >> 4) & 0xF, 16)).append(Character.forDigit(b & 0xF, 16)); - } - return h.toString(); - } - - private static String fromHex(String h) { - if (h.length() % 2 != 0) { - throw new IllegalArgumentException("not a valid rest source token: odd-length hex body"); - } - byte[] bytes = new byte[h.length() / 2]; - for (int i = 0; i < bytes.length; i++) { - bytes[i] = - (byte) - ((Character.digit(h.charAt(2 * i), 16) << 4) - + Character.digit(h.charAt(2 * i + 1), 16)); - } - return new String(bytes, StandardCharsets.UTF_8); - } - - /** - * The validated spec for a {@code rest} command: an allow-listed read-only endpoint plus optional - * count/timeout/query args. Lives in core so the parser (encode) and the storage engine (decode) - * share it without a cross-module dependency. - */ - @Getter - @RequiredArgsConstructor - public static class RestSpec { - private final String endpoint; - private final Map args; - private final Integer count; - private final String timeout; - } - /** * Compose system mapping table. * diff --git a/docs/category.json b/docs/category.json index a8665b82eac..7296b089650 100644 --- a/docs/category.json +++ b/docs/category.json @@ -35,7 +35,6 @@ "user/ppl/cmd/rename.md", "user/ppl/cmd/multisearch.md", "user/ppl/cmd/replace.md", - "user/ppl/cmd/rest.md", "user/ppl/cmd/rex.md", "user/ppl/cmd/search.md", "user/ppl/cmd/showdatasources.md", diff --git a/docs/user/ppl/cmd/rest.md b/docs/user/ppl/cmd/rest.md deleted file mode 100644 index a9761aa87e8..00000000000 --- a/docs/user/ppl/cmd/rest.md +++ /dev/null @@ -1,94 +0,0 @@ -# rest - -The `rest` command is a leading command that reads an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) and emits the response as PPL rows. Its rows come from the endpoint dispatch, not from an index, so `rest` appears at the start of a query. - -> **Note**: The `rest` command is supported only on the Calcite query engine (`plugins.calcite.enabled=true`). Each endpoint has a fixed output schema, and the dispatch runs under the caller's security context, so a user who cannot call an endpoint directly cannot call it through `rest`. The command is read-only; mutating and non-allow-listed endpoints are rejected. Each endpoint requires the same cluster-monitor privilege as calling it natively, so `rest` grants no extra access. Some allow-listed endpoints surface operational metadata (for example `/_cat/nodes` exposes node addresses and resource utilization, `/_cat/plugins` the installed plugin inventory, and `/_cluster/state` cluster-state identifiers); this is a deliberate, read-only, monitor-privileged trade-off. `/_cluster/settings` is redacted with the node's setting filter so `Property.Filtered` keys are not surfaced. - -## Enabling the command - -The `rest` command is **disabled by default**: `plugins.ppl.rest.allowed_endpoints` defaults to an empty list, so every endpoint is rejected until a deployment explicitly opts in. Enable specific endpoints by setting the allow-list (a node-level setting, so it is applied at node startup and cannot be changed at runtime): - -```yaml -plugins.ppl.rest.allowed_endpoints: ["/_cluster/health", "/_cat/nodes"] -``` - -Use `["*"]` to allow every endpoint in the curated list below. An empty list (the default) disables the command entirely. - -The `rest` command also supports optional response redaction of network identifiers (IPv4/IPv6 addresses, `inet[...]` forms, EC2-style host names, and availability-zone names) in `/_cat/*` and `/_cluster/settings` cell values, controlled by `plugins.ppl.rest.redaction.enabled` (a node-level setting, default `false`). Managed deployments that must not expose host topology should set it to `true`. - -## Syntax - -The `rest` command has the following syntax: - -```syntax -rest [count=] [timeout=] [= ...] -``` - -## Parameters - -The `rest` command supports the following parameters. - -| Parameter | Required/Optional | Description | -| --- | --- | --- | -| `` | Required | An allow-listed, read-only endpoint path (see the allow-list below), for example `/_cluster/health`. | -| `count=` | Optional | Caps the number of emitted rows. | -| `timeout=` | Optional | Reserved for forward compatibility. It is currently rejected with a clear error, because a single uniform timeout does not map cleanly across the different endpoints. | -| `=` | Optional | Endpoint query arguments, validated per endpoint by both key and value (for example `local=true` for `/_cluster/health`, `health=green` for `/_cat/indices`, `expand_wildcards=open` for `/_resolve/index`). | - -## Allow-list - -`rest` resolves only an explicit, curated set of read-only endpoints. Anything outside the list, including any mutating endpoint, is rejected with a clear error. - -| Endpoint | Output columns | Accepted args | -| --- | --- | --- | -| `/_cluster/health` | `cluster_name` (string), `status` (string), `number_of_nodes` (integer), `number_of_data_nodes` (integer), `active_primary_shards` (integer), `active_shards` (integer), `relocating_shards` (integer), `initializing_shards` (integer), `unassigned_shards` (integer), `timed_out` (boolean) | `local` | -| `/_cluster/state` | `cluster_name` (string), `state_uuid` (string), `version` (long), `cluster_manager_node` (string) | (none) | -| `/_cluster/settings` | `setting` (string), `value` (string), `tier` (string) | (none) | -| `/_cat/indices` | `index` (string), `health` (string), `pri` (integer), `rep` (integer), `active_shards` (integer) | `health` | -| `/_cat/nodes` | `name` (string), `ip` (string), `node_role` (string), `heap_percent` (integer), `ram_percent` (integer), `cpu` (integer) | (none) | -| `/_cat/cluster_manager` | `id` (string), `host` (string), `ip` (string), `node` (string) | (none) | -| `/_cat/plugins` | `name` (string), `component` (string), `version` (string) | (none) | -| `/_cat/shards` | `index` (string), `shard` (integer), `prirep` (string), `state` (string), `node` (string) | (none) | -| `/_resolve/index` | `name` (string), `type` (string) | `expand_wildcards` | - -## Example 1: Counting the nodes in the cluster - -The following query reads cluster health and projects a column that is deterministic on a single-node cluster: - -```ppl -| rest '/_cluster/health' | fields number_of_nodes -``` - -The query returns the following results: - -```text -fetched rows / total rows = 1/1 -+-----------------+ -| number_of_nodes | -|-----------------| -| 1 | -+-----------------+ -``` - -`/_cluster/health` also exposes `status`, `active_shards`, and the other columns listed in the allow-list, which you can project and filter the same way. - -## Example 2: Composing downstream commands over a cat endpoint - -The `rest` row source composes with downstream `where`, `sort`, `stats`, and `fields` exactly like an index scan. The following query reads `/_cat/cluster_manager` and counts the rows: - -```ppl -| rest '/_cat/cluster_manager' | stats count() as managers -``` - -The query returns the following results: - -```text -fetched rows / total rows = 1/1 -+----------+ -| managers | -|----------| -| 1 | -+----------+ -``` - -For example, `| rest '/_cat/indices' | where health = 'green' | sort index | fields index, health, pri` lists green indexes; the projected columns come from the endpoint's fixed schema. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 939684c0ecc..37947113800 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -74,7 +74,6 @@ source=accounts | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | | [multisearch command](cmd/multisearch.md) | 3.4 | experimental (since 3.4) | Execute multiple search queries and combine their results. | | [union command](cmd/union.md) | 3.7 | experimental (since 3.7) | Combine results from multiple datasets using UNION ALL semantics. | -| [rest command](cmd/rest.md) | 3.8 | experimental (since 3.8) | Read an allow-listed, read-only in-cluster management endpoint (cluster/cat/nodes) as rows. Calcite engine only. | | [ml command](cmd/ml.md) | 2.5 | stable (since 2.5) | Apply machine learning algorithms to analyze data. | | [kmeans command](cmd/kmeans.md) | 1.3 | stable (since 1.3) | Apply the kmeans algorithm on the search result returned by a PPL command. | | [ad command](cmd/ad.md) | 1.3 | deprecated (since 2.5) | Apply Random Cut Forest algorithm on the search result returned by a PPL command. | diff --git a/doctest/build.gradle b/doctest/build.gradle index 1ac658457dc..cce64170f56 100644 --- a/doctest/build.gradle +++ b/doctest/build.gradle @@ -205,11 +205,6 @@ testClusters { plugin(getJobSchedulerPlugin()) plugin ':opensearch-sql-plugin' testDistribution = 'archive' - // The rest command is disabled by default (empty allow-list). Opt the doctest cluster into - // the registered endpoints so the rest.md examples run against an enabled command. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } } tasks.register("runRestTestCluster", RunTask) { diff --git a/integ-test/build.gradle b/integ-test/build.gradle index c18fa6e37f6..1435d1d499d 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -387,11 +387,6 @@ testClusters { plugin(getGeoSpatialPlugin()) plugin ":opensearch-sql-plugin" setting "plugins.query.datasources.encryption.masterkey", "1234567812345678" - // The rest command is disabled by default (empty allow-list). Opt this cluster into the - // registered endpoints so the rest integration tests exercise the enabled path. A literal - // "*" cannot be used because a bare * is a YAML alias indicator in opensearch.yml. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } yamlRestTest { testDistribution = 'archive' @@ -410,9 +405,6 @@ testClusters { testDistribution = 'archive' plugin(getJobSchedulerPlugin()) plugin ":opensearch-sql-plugin" - // Opt into the rest endpoints (disabled by default) so RestCommandSecurityIT runs. - setting 'plugins.ppl.rest.allowed_endpoints', - '/_cluster/health,/_cluster/state,/_cluster/settings,/_cat/indices,/_cat/nodes,/_cat/cluster_manager,/_cat/plugins,/_cat/shards,/_resolve/index' } remoteIntegTestWithSecurity { testDistribution = 'archive' @@ -427,8 +419,6 @@ testClusters { plugin(getArrowFlightRpcPlugin()) plugin(getAnalyticsEnginePlugin()) plugin ":opensearch-sql-plugin" - // Composite-default cluster: PPL queries route to the analytics engine unless excluded. - setting 'cluster.pluggable.dataformat', 'composite' } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 801d56fd49d..907f91fa98e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -58,7 +58,6 @@ CalciteObjectFieldOperateIT.class, CalciteOperatorIT.class, CalciteParseCommandIT.class, - CalcitePPLRestIT.class, CalcitePPLAggregationIT.class, CalcitePPLAppendcolIT.class, CalcitePPLAppendCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 4ab1d40a5af..b78a71e534c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -66,15 +66,6 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } - // Only for Calcite: the rest row source explains as a CalciteScannableCatalogScan. - @Test - public void explainRestCommand() throws IOException { - String result = explainQueryToString("| rest '/_cluster/health' | fields status"); - Assert.assertTrue( - "Expected a rest scan node in the explain output, got: " + result, - result.contains("CatalogScan")); - } - @Override @Ignore("test only in v2") public void testExplainModeUnsupportedInV2() throws IOException {} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java deleted file mode 100644 index 70ee4ef0de6..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLRestIT.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.calcite.remote; - -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; - -import java.io.IOException; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; -import org.opensearch.client.Request; -import org.opensearch.client.ResponseException; -import org.opensearch.sql.ppl.PPLIntegTestCase; - -/** - * Integration tests for the {@code rest} leading command on the Calcite path. Uses {@code - * /_cluster/health} as the deterministic, single-row endpoint on a single-node test cluster. Also - * verifies that a non-allow-listed / mutating endpoint is refused. - */ -public class CalcitePPLRestIT extends PPLIntegTestCase { - - @Override - public void init() throws Exception { - super.init(); - enableCalcite(); - } - - @Test - public void testRestClusterHealthSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); - verifySchema(result, schema("status", "string"), schema("number_of_nodes", "int")); - } - - @Test - public void testRestClusterHealthDataRows() throws IOException { - // Single-node test cluster: exactly one node, status is green or yellow. - JSONObject result = executeQuery("| rest '/_cluster/health' | fields number_of_nodes"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestRejectsNonAllowListedEndpoint() throws IOException { - assertRestBadRequest("| rest '/_cluster/reroute'", "allow-list"); - } - - @Test - public void testRestRejectsEmptyEndpoint() throws IOException { - assertRestBadRequest("| rest ''", "non-empty path"); - } - - @Test - public void testRestRejectsDisallowedArg() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' h='name'", "does not accept arg"); - } - - @Test - public void testRestRejectsNegativeCount() throws IOException { - assertRestBadRequest("| rest '/_cat/nodes' count=-1", "non-negative"); - } - - @Test - public void testRestRejectsTimeoutArg() throws IOException { - assertRestBadRequest("| rest '/_cluster/health' timeout='5s'", "timeout"); - } - - /** - * Assert a {@code rest} query is refused as a client error: HTTP 400 (not a 500 system error) - * with the given substring in the response body. Covers allow-list and bad-argument rejection. - */ - private void assertRestBadRequest(String query, String expectedSubstring) { - ResponseException e = - org.junit.Assert.assertThrows(ResponseException.class, () -> executeQuery(query)); - org.junit.Assert.assertEquals(400, e.getResponse().getStatusLine().getStatusCode()); - org.junit.Assert.assertTrue( - "expected [" + expectedSubstring + "] in response body: " + e.getMessage(), - e.getMessage().contains(expectedSubstring)); - } - - @Test - public void testRestCatIndicesSchema() throws IOException { - // Schema is fixed by the registry, independent of how many indices exist. - JSONObject result = executeQuery("| rest '/_cat/indices' | fields index, health"); - verifySchema(result, schema("index", "string"), schema("health", "string")); - } - - @Test - public void testRestCatIndicesReturnsCreatedIndex() throws IOException { - // Create a known index, then confirm rest surfaces it and downstream where/fields compose. - client().performRequest(new Request("PUT", "/rest_cat_test")); - JSONObject result = - executeQuery("| rest '/_cat/indices' | where index = 'rest_cat_test' | fields index"); - verifyDataRows(result, rows("rest_cat_test")); - } - - @Test - public void testRestCatNodesSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | fields name, cpu"); - verifySchema(result, schema("name", "string"), schema("cpu", "int")); - } - - @Test - public void testRestCatNodesSingleNode() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/nodes' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatClusterManagerSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | fields node, id"); - verifySchema(result, schema("node", "string"), schema("id", "string")); - } - - @Test - public void testRestCatClusterManagerSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/cluster_manager' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatPluginsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/plugins' | fields component, version"); - verifySchema(result, schema("component", "string"), schema("version", "string")); - } - - @Test - public void testRestCatShardsSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_cat/shards' | fields index, shard, state"); - verifySchema( - result, schema("index", "string"), schema("shard", "int"), schema("state", "string")); - } - - @Test - public void testRestClusterStateSchema() throws IOException { - // Assert the string columns; version is the LONG epoch column. - JSONObject result = - executeQuery("| rest '/_cluster/state' | fields cluster_name, cluster_manager_node"); - verifySchema( - result, schema("cluster_name", "string"), schema("cluster_manager_node", "string")); - } - - @Test - public void testRestClusterStateSingleRow() throws IOException { - JSONObject result = executeQuery("| rest '/_cluster/state' | stats count() as cnt"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestClusterSettingsSchema() throws IOException { - // Schema is registry-fixed regardless of how many settings are configured. - JSONObject result = executeQuery("| rest '/_cluster/settings' | fields setting, value, tier"); - verifySchema( - result, schema("setting", "string"), schema("value", "string"), schema("tier", "string")); - } - - @Test - public void testRestResolveIndexSchema() throws IOException { - JSONObject result = executeQuery("| rest '/_resolve/index' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); - } - - @Test - public void testRestResolveIndexSurfacesCreatedIndex() throws IOException { - client().performRequest(new Request("PUT", "/rest_resolve_test")); - JSONObject result = - executeQuery( - "| rest '/_resolve/index' | where name = 'rest_resolve_test' | fields name, type"); - verifyDataRows(result, rows("rest_resolve_test", "index")); - } - - // ---- get-arg server-side filtering (health, expand_wildcards, local) ---- - - @Test - public void testRestClusterHealthLocalArg() throws IOException { - // local=true reads health from the local node; on a single-node cluster the row is unchanged. - JSONObject result = - executeQuery("| rest '/_cluster/health' local='true' | fields number_of_nodes"); - verifyDataRows(result, rows(1)); - } - - @Test - public void testRestCatIndicesHealthFilterReturnsNoRed() throws IOException { - // health filters rows server-side; a healthy cluster has no red indices, so count is 0. - JSONObject result = executeQuery("| rest '/_cat/indices' health='red' | stats count() as cnt"); - verifyDataRows(result, rows(0)); - } - - @Test - public void testRestResolveIndexExpandWildcardsArg() throws IOException { - // expand_wildcards is applied to the resolve request; schema stays fixed and the call succeeds. - JSONObject result = - executeQuery("| rest '/_resolve/index' expand_wildcards='open' | fields name, type"); - verifySchema(result, schema("name", "string"), schema("type", "string")); - } - - @Test - public void testRestRejectsDroppedLevelArg() throws IOException { - // level was dropped from the allow-list (no-op against the fixed health schema). - assertRestBadRequest("| rest '/_cluster/health' level='indices'", "does not accept arg"); - } - - @Test - public void testRestRejectsBadArgValue() throws IOException { - assertRestBadRequest("| rest '/_cat/indices' health='purple'", "unsupported value"); - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java index b5c08cc8ad0..f6ec903c395 100644 --- a/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/plugin/AnalyticsEngineCompatIT.java @@ -5,13 +5,9 @@ package org.opensearch.sql.plugin; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; import java.io.IOException; -import org.json.JSONArray; -import org.json.JSONObject; import org.junit.Before; import org.junit.Test; import org.opensearch.client.Request; @@ -60,31 +56,4 @@ public void testClusterStarted() { // If the cluster booted with analytics-engine present, all plugins loaded without classloader // errors. The assumption above guarantees we only assert this where it is meaningful. } - - /** - * The {@code rest} row source is a Calcite Enumerable/Scannable scan with no backing index, so it - * is never routed to the analytics (DataFusion) engine. This pins that {@code rest} returns its - * fixed schema and correct data unchanged when the analytics-engine plugin is present. - */ - @Test - public void testRestCommandUnaffectedByAnalyticsEngine() throws IOException { - Request request = new Request("POST", "/_plugins/_ppl"); - request.setJsonEntity( - "{\"query\": \"| rest '/_cluster/health' | fields status, number_of_nodes\"}"); - Response response = client().performRequest(request); - assertEquals(200, response.getStatusLine().getStatusCode()); - - JSONObject result = new JSONObject(TestUtils.getResponseBody(response, true)); - - JSONArray schema = result.getJSONArray("schema"); - assertEquals(2, schema.length()); - assertEquals("status", schema.getJSONObject(0).getString("name")); - assertEquals("string", schema.getJSONObject(0).getString("type")); - assertEquals("number_of_nodes", schema.getJSONObject(1).getString("name")); - assertEquals("int", schema.getJSONObject(1).getString("type")); - - JSONArray datarows = result.getJSONArray("datarows"); - assertEquals(1, datarows.length()); - assertTrue(datarows.getJSONArray(0).getInt(1) >= 1); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 2ccda31eea7..837865a3585 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -32,19 +32,6 @@ public void init() throws Exception { loadIndex(Index.GRAPH_EMPLOYEES); } - @Test - public void testRest() throws IOException { - JSONObject result; - try { - result = executeQuery("| rest '/_cluster/health' | fields status, number_of_nodes"); - } catch (ResponseException e) { - result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); - } - if (isCalciteEnabled()) { - assertFalse(result.getJSONArray("datarows").isEmpty()); - } - } - @Test public void testJoin() throws IOException { JSONObject result; diff --git a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java b/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java deleted file mode 100644 index 042a3aefbf7..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/security/RestCommandSecurityIT.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.security; - -import static org.opensearch.sql.util.MatcherUtils.columnName; -import static org.opensearch.sql.util.MatcherUtils.verifyColumn; - -import java.io.IOException; -import java.util.HashSet; -import java.util.Set; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Test; -import org.opensearch.client.Request; -import org.opensearch.client.ResponseException; -import org.opensearch.sql.legacy.TestUtils; - -/** - * Integration tests that verify the rest command is subject to the security plugin fine grained - * access control. The command dispatches standard transport actions under the caller identity, so - * the security ActionFilter authorizes each one by action name. A caller without the required - * cluster monitor privilege is denied the cat and cluster endpoints, a caller holding the privilege - * can run them, and the resolve index endpoint requires the resolve index privilege because the - * command resolves all indices. - */ -public class RestCommandSecurityIT extends SecurityTestBase { - - private static final String ALPHA_INDEX = "rest_sec_alpha"; - private static final String BETA_INDEX = "rest_sec_beta"; - - private static final String MONITOR_USER = "rest_monitor_user"; - private static final String MONITOR_ROLE = "rest_monitor_role"; - - private static final String NO_MONITOR_USER = "rest_no_monitor_user"; - private static final String NO_MONITOR_ROLE = "rest_no_monitor_role"; - - @Override - protected void init() throws Exception { - super.init(); - setupRolesUsersAndIndices(); - enableCalcite(); - // rest is Calcite only, so a V2 fallback would replace the security denial with an unsupported - // command error. Disable fallback so the denial reason surfaces to the caller. - disallowCalciteFallback(); - } - - private void setupRolesUsersAndIndices() throws IOException { - createIndexIfAbsent(ALPHA_INDEX); - createIndexIfAbsent(BETA_INDEX); - - createRoleWithPermissions( - MONITOR_ROLE, - "*", - new String[] { - "cluster:admin/opensearch/ppl", - "cluster:monitor/health", - "cluster:monitor/state", - "cluster:monitor/nodes/stats", - "cluster:monitor/nodes/info" - }, - new String[] {"indices:admin/resolve/index"}); - createUser(MONITOR_USER, MONITOR_ROLE); - - createRoleWithPermissions( - NO_MONITOR_ROLE, - ALPHA_INDEX, - new String[] {"cluster:admin/opensearch/ppl"}, - new String[] {"indices:data/read/search*"}); - createUser(NO_MONITOR_USER, NO_MONITOR_ROLE); - } - - @Test - public void monitorUserCanRunCatNodes() throws IOException { - JSONObject result = executeQueryAsUser("| rest '/_cat/nodes' | fields name", MONITOR_USER); - verifyColumn(result, columnName("name")); - } - - @Test - public void monitorUserCanResolveIndex() throws IOException { - JSONObject result = - executeQueryAsUser("| rest '/_resolve/index' | fields name, type", MONITOR_USER); - Set names = resolvedNames(result); - assertTrue("resolve should list authorized indices: " + names, names.contains(ALPHA_INDEX)); - assertTrue("resolve should list authorized indices: " + names, names.contains(BETA_INDEX)); - } - - @Test - public void userWithoutClusterMonitorCannotRunCatNodes() throws IOException { - assertDenied( - "| rest '/_cat/nodes' | fields name", NO_MONITOR_USER, "cluster:monitor/nodes/stats"); - } - - @Test - public void userWithoutClusterMonitorCannotRunClusterState() throws IOException { - assertDenied( - "| rest '/_cluster/state' | fields cluster_name", NO_MONITOR_USER, "cluster:monitor/state"); - } - - @Test - public void userWithoutResolvePrivilegeCannotResolveIndex() throws IOException { - assertDenied( - "| rest '/_resolve/index' | fields name, type", - NO_MONITOR_USER, - "indices:admin/resolve/index"); - } - - /** - * Asserts the query is rejected for a caller lacking the privilege. A denied transport action on - * the Calcite only rest path surfaces as a client or server error whose body carries the security - * denial reason, so this checks the denial signal rather than a fixed status code. - */ - private void assertDenied(String query, String user, String deniedAction) throws IOException { - try { - executeQueryAsUser(query, user); - fail("Expected a permission denial for user without privilege: " + user); - } catch (ResponseException e) { - int status = e.getResponse().getStatusLine().getStatusCode(); - String body = TestUtils.getResponseBody(e.getResponse(), false); - assertTrue("Expected an error status, got " + status, status >= 400); - assertTrue( - "Response should indicate a permission denial. Status " + status + ", body: " + body, - body.contains("no permissions") - || body.contains("Forbidden") - || body.contains("security_exception") - || body.contains(deniedAction)); - } - } - - private void createIndexIfAbsent(String name) throws IOException { - Request request = new Request("PUT", "/" + name); - request.setJsonEntity( - "{ \"settings\": { \"number_of_shards\": 1, \"number_of_replicas\": 0 } }"); - try { - client().performRequest(request); - } catch (ResponseException e) { - String body = TestUtils.getResponseBody(e.getResponse(), false); - if (!body.contains("resource_already_exists_exception")) { - throw e; - } - } - } - - private Set resolvedNames(JSONObject result) { - Set names = new HashSet<>(); - JSONArray datarows = result.getJSONArray("datarows"); - for (int i = 0; i < datarows.length(); i++) { - names.add(datarows.getJSONArray(i).getString(0)); - } - return names; - } -} diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java index c10bc474bae..f3c3046f41e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/VectorSearchIT.java @@ -28,7 +28,6 @@ public class VectorSearchIT extends SQLIntegTestCase { @Override protected void init() throws Exception { - super.init(); loadIndex(Index.ACCOUNT); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index 3b9c3619521..68350c5a0fd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -114,105 +114,4 @@ public interface OpenSearchClient { * @param deletePitRequest Delete Point In Time request */ void deletePit(DeletePitRequest deletePitRequest); - - /** - * Read-only cluster health snapshot for the {@code rest} command (backs {@code - * /_cluster/health}). Returns a single flattened row of health fields. Runs under the caller's - * security thread-context; performs no privilege escalation and mutates nothing. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of health field name to value - */ - default Map clusterHealth(Map params) { - throw new UnsupportedOperationException("clusterHealth is not supported by this client"); - } - - /** - * Read-only cat-indices listing for the {@code rest} command (backs {@code /_cat/indices}). One - * map per index. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per index - */ - default List> catIndices(Map params) { - throw new UnsupportedOperationException("catIndices is not supported by this client"); - } - - /** - * Read-only cat-nodes listing for the {@code rest} command (backs {@code /_cat/nodes}). One map - * per node with resource state. Runs under the caller's security thread-context; read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per node - */ - default List> catNodes(Map params) { - throw new UnsupportedOperationException("catNodes is not supported by this client"); - } - - /** - * Read-only cat-cluster_manager listing for the {@code rest} command (backs {@code - * /_cat/cluster_manager}). Single map identifying the elected cluster manager. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map describing the cluster manager node - */ - default List> catClusterManager(Map params) { - throw new UnsupportedOperationException("catClusterManager is not supported by this client"); - } - - /** - * Read-only cat-plugins listing for the {@code rest} command (backs {@code /_cat/plugins}). One - * map per installed plugin per node. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per plugin - */ - default List> catPlugins(Map params) { - throw new UnsupportedOperationException("catPlugins is not supported by this client"); - } - - /** - * Read-only cat-shards listing for the {@code rest} command (backs {@code /_cat/shards}). One map - * per shard. Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per shard - */ - default List> catShards(Map params) { - throw new UnsupportedOperationException("catShards is not supported by this client"); - } - - /** - * Read-only cluster-state epoch projection for the {@code rest} command (backs {@code - * /_cluster/state}). Single flattened row (cluster_name, state_uuid, version, - * cluster_manager_node). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return a single map of cluster-state field name to value - */ - default Map clusterState(Map params) { - throw new UnsupportedOperationException("clusterState is not supported by this client"); - } - - /** - * Read-only cluster-settings listing for the {@code rest} command (backs {@code - * /_cluster/settings}). One map per configured setting (setting, value, tier). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per setting - */ - default List> clusterSettings(Map params) { - throw new UnsupportedOperationException("clusterSettings is not supported by this client"); - } - - /** - * Read-only resolve-index listing for the {@code rest} command (backs {@code /_resolve/index}). - * One map per resolved index, alias, or data stream (name, type). Read-only. - * - * @param params endpoint query args (already allow-list-validated) - * @return one map of column name to value per resolved name - */ - default List> resolveIndex(Map params) { - throw new UnsupportedOperationException("resolveIndex is not supported by this client"); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 080a8627894..b491f38ef80 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -18,8 +18,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.opensearch.OpenSearchSecurityException; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse; @@ -27,7 +25,6 @@ import org.opensearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.search.*; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; @@ -288,274 +285,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.admin().cluster().health(request).actionGet(); - return flattenHealth(response); - } - - @Override - public List> catIndices(Map params) { - ClusterHealthResponse response = - client.admin().cluster().health(new ClusterHealthRequest()).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new java.util.LinkedHashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } - - @Override - public List> catNodes(Map params) { - org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest statsRequest = - new org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest(); - statsRequest.all(); - org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse response = - client.admin().cluster().nodesStats(statsRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.stats.NodeStats ns : response.getNodes()) { - org.opensearch.cluster.node.DiscoveryNode node = ns.getNode(); - Map row = new java.util.LinkedHashMap<>(); - row.put("name", node.getName()); - row.put("ip", node.getHostAddress()); - row.put( - "node_role", - node.getRoles().stream() - .map(org.opensearch.cluster.node.DiscoveryNodeRole::roleName) - .sorted() - .collect(java.util.stream.Collectors.joining(","))); - row.put( - "heap_percent", - ns.getJvm() == null || ns.getJvm().getMem() == null - ? null - : (int) ns.getJvm().getMem().getHeapUsedPercent()); - row.put( - "ram_percent", - ns.getOs() == null || ns.getOs().getMem() == null - ? null - : (int) ns.getOs().getMem().getUsedPercent()); - row.put( - "cpu", - ns.getProcess() == null || ns.getProcess().getCpu() == null - ? null - : (int) ns.getProcess().getCpu().getPercent()); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - List> rows = new java.util.ArrayList<>(); - if (cm != null) { - Map row = new java.util.LinkedHashMap<>(); - row.put("id", cm.getId()); - row.put("host", cm.getHostName()); - row.put("ip", cm.getHostAddress()); - row.put("node", cm.getName()); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - org.opensearch.action.admin.cluster.node.info.NodesInfoRequest infoRequest = - new org.opensearch.action.admin.cluster.node.info.NodesInfoRequest(); - infoRequest.all(); - org.opensearch.action.admin.cluster.node.info.NodesInfoResponse response = - client.admin().cluster().nodesInfo(infoRequest).actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.cluster.node.info.NodeInfo info : response.getNodes()) { - org.opensearch.action.admin.cluster.node.info.PluginsAndModules plugins = - info.getInfo(org.opensearch.action.admin.cluster.node.info.PluginsAndModules.class); - if (plugins == null) { - continue; - } - for (org.opensearch.plugins.PluginInfo pi : plugins.getPluginInfos()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", info.getNode().getName()); - row.put("component", pi.getName()); - row.put("version", pi.getVersion()); - rows.add(row); - } - } - return rows; - } - - @Override - public List> catShards(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - org.opensearch.cluster.node.DiscoveryNodes nodes = response.getState().nodes(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.cluster.routing.ShardRouting sr : - response.getState().getRoutingTable().allShards()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("index", sr.getIndexName()); - row.put("shard", sr.id()); - row.put("prirep", sr.primary() ? "p" : "r"); - row.put("state", sr.state().name()); - org.opensearch.cluster.node.DiscoveryNode n = - sr.currentNodeId() == null ? null : nodes.get(sr.currentNodeId()); - row.put("node", n == null ? null : n.getName()); - rows.add(row); - } - return rows; - } - - @Override - public Map clusterState(Map params) { - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName().value()); - row.put("state_uuid", response.getState().stateUUID()); - row.put("version", response.getState().version()); - org.opensearch.cluster.node.DiscoveryNode cm = - response.getState().nodes().getClusterManagerNode(); - row.put("cluster_manager_node", cm == null ? null : cm.getName()); - return row; - } - - @Override - public List> clusterSettings(Map params) { - // The transport path has no SettingsFilter of its own; it is published from - // SQLPlugin#getRestHandlers at startup. Fail closed before fetching so we never read settings - // into memory when we cannot redact them, matching native GET /_cluster/settings. - org.opensearch.common.settings.SettingsFilter filter = - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.get(); - if (filter == null) { - throw new IllegalStateException( - "cluster settings redaction filter is not initialized; refusing to return unredacted" - + " settings"); - } - org.opensearch.action.admin.cluster.state.ClusterStateResponse response = - client - .admin() - .cluster() - .state(new org.opensearch.action.admin.cluster.state.ClusterStateRequest()) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - org.opensearch.common.settings.Settings persistent = - filter.filter(response.getState().metadata().persistentSettings()); - org.opensearch.common.settings.Settings transientSettings = - filter.filter(response.getState().metadata().transientSettings()); - collectSettings(persistent, "persistent", rows); - collectSettings(transientSettings, "transient", rows); - return rows; - } - - private void collectSettings( - org.opensearch.common.settings.Settings settings, - String tier, - List> rows) { - if (settings == null) { - return; - } - for (String key : settings.keySet()) { - Map row = new java.util.LinkedHashMap<>(); - row.put("setting", key); - String value = settings.get(key); - if (value == null) { - // List-valued settings return null from get(); fall back to the joined list form. - java.util.List list = settings.getAsList(key); - value = list.isEmpty() ? null : String.join(",", list); - } - row.put("value", value); - row.put("tier", tier); - rows.add(row); - } - } - - @Override - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request request = - expandWildcards == null - ? new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}) - : new org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request( - new String[] {"*"}, - org.opensearch.action.support.IndicesOptions.fromParameters( - expandWildcards, - null, - null, - null, - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Request - .DEFAULT_INDICES_OPTIONS)); - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.Response response = - client - .execute( - org.opensearch.action.admin.indices.resolve.ResolveIndexAction.INSTANCE, request) - .actionGet(); - List> rows = new java.util.ArrayList<>(); - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedIndex idx : - response.getIndices()) { - rows.add(resolveRow(idx.getName(), "index")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedAlias alias : - response.getAliases()) { - rows.add(resolveRow(alias.getName(), "alias")); - } - for (org.opensearch.action.admin.indices.resolve.ResolveIndexAction.ResolvedDataStream ds : - response.getDataStreams()) { - rows.add(resolveRow(ds.getName(), "data_stream")); - } - return rows; - } - - private Map resolveRow(String name, String type) { - Map row = new java.util.LinkedHashMap<>(); - row.put("name", name); - row.put("type", type); - return row; - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new java.util.LinkedHashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(java.util.Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index e98c5bf95f4..f369c0003b8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -8,19 +8,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; -import org.opensearch.action.admin.cluster.health.ClusterHealthRequest; -import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; @@ -32,7 +28,6 @@ import org.opensearch.client.indices.GetIndexResponse; import org.opensearch.client.indices.GetMappingsRequest; import org.opensearch.client.indices.GetMappingsResponse; -import org.opensearch.cluster.health.ClusterIndexHealth; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexNotFoundException; @@ -277,267 +272,4 @@ public void deletePit(DeletePitRequest deletePitRequest) { "Error occurred while deleting PIT for internal plugin operation", e); } } - - @Override - public Map clusterHealth(Map params) { - try { - ClusterHealthRequest request = new ClusterHealthRequest(); - if (params != null && Boolean.parseBoolean(params.get("local"))) { - request.local(true); - } - ClusterHealthResponse response = client.cluster().health(request, RequestOptions.DEFAULT); - return flattenHealth(response); - } catch (IOException e) { - throw new IllegalStateException("Failed to get cluster health", e); - } - } - - @Override - public List> catIndices(Map params) { - try { - ClusterHealthResponse response = - client.cluster().health(new ClusterHealthRequest(), RequestOptions.DEFAULT); - List> rows = new ArrayList<>(); - for (Map.Entry entry : response.getIndices().entrySet()) { - ClusterIndexHealth health = entry.getValue(); - Map row = new HashMap<>(); - row.put("index", entry.getKey()); - row.put("health", health.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("pri", health.getNumberOfShards()); - row.put("rep", health.getNumberOfReplicas()); - row.put("active_shards", health.getActiveShards()); - rows.add(row); - } - String healthFilter = params == null ? null : params.get("health"); - if (healthFilter != null) { - rows.removeIf(r -> !healthFilter.equalsIgnoreCase(String.valueOf(r.get("health")))); - } - return rows; - } catch (IOException e) { - throw new IllegalStateException("Failed to get cat indices", e); - } - } - - @Override - public List> catNodes(Map params) { - List> raw = - catJson("/_cat/nodes", "name,ip,node.role,heap.percent,ram.percent,cpu"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("ip", r.get("ip")); - row.put("node_role", r.get("node.role")); - row.put("heap_percent", asInt(r.get("heap.percent"))); - row.put("ram_percent", asInt(r.get("ram.percent"))); - row.put("cpu", asInt(r.get("cpu"))); - rows.add(row); - } - return rows; - } - - @Override - public List> catClusterManager(Map params) { - List> raw = catJson("/_cat/cluster_manager", "id,host,ip,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("id", r.get("id")); - row.put("host", r.get("host")); - row.put("ip", r.get("ip")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - @Override - public List> catPlugins(Map params) { - List> raw = catJson("/_cat/plugins", "name,component,version"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("name", r.get("name")); - row.put("component", r.get("component")); - row.put("version", r.get("version")); - rows.add(row); - } - return rows; - } - - @Override - public List> catShards(Map params) { - List> raw = catJson("/_cat/shards", "index,shard,prirep,state,node"); - List> rows = new ArrayList<>(); - for (Map r : raw) { - Map row = new HashMap<>(); - row.put("index", r.get("index")); - row.put("shard", asInt(r.get("shard"))); - row.put("prirep", r.get("prirep")); - row.put("state", r.get("state")); - row.put("node", r.get("node")); - rows.add(row); - } - return rows; - } - - /** Standalone-mode helper: GET a _cat endpoint as JSON via the low-level client. */ - @SuppressWarnings("unchecked") - private List> catJson(String path, String columns) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - request.addParameter("format", "json"); - request.addParameter("h", columns); - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - List list = parser.list(); - List> rows = new ArrayList<>(); - for (Object o : list) { - rows.add((Map) o); - } - return rows; - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Integer asInt(Object value) { - if (value == null) { - return null; - } - try { - return (int) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public Map clusterState(Map params) { - Map state = - getJsonMap( - "/_cluster/state/master_node,version,metadata,nodes", - // nodes.*.name resolves the manager id to a name without over-fetching node IPs. - Map.of( - "filter_path", - "cluster_name,state_uuid,version,cluster_manager_node,nodes.*.name")); - Map row = new HashMap<>(); - row.put("cluster_name", state.get("cluster_name")); - row.put("state_uuid", state.get("state_uuid")); - row.put("version", asLong(state.get("version"))); - Object cmId = state.get("cluster_manager_node"); - String cmName = null; - Object nodes = state.get("nodes"); - if (cmId != null && nodes instanceof Map) { - Object n = ((Map) nodes).get(cmId.toString()); - if (n instanceof Map) { - Object name = ((Map) n).get("name"); - cmName = name == null ? null : name.toString(); - } - } - row.put("cluster_manager_node", cmName); - return row; - } - - @Override - @SuppressWarnings("unchecked") - public List> clusterSettings(Map params) { - Map body = getJsonMap("/_cluster/settings", Map.of("flat_settings", "true")); - List> rows = new ArrayList<>(); - for (String tier : new String[] {"persistent", "transient"}) { - Object section = body.get(tier); - if (section instanceof Map) { - for (Map.Entry e : ((Map) section).entrySet()) { - Map row = new HashMap<>(); - row.put("setting", e.getKey()); - row.put("value", e.getValue() == null ? null : e.getValue().toString()); - row.put("tier", tier); - rows.add(row); - } - } - } - return rows; - } - - /** Standalone-mode helper: GET a JSON-object endpoint via the low-level client. */ - @SuppressWarnings("unchecked") - private Map getJsonMap(String path, Map params) { - try { - org.opensearch.client.Request request = new org.opensearch.client.Request("GET", path); - if (params != null) { - params.forEach(request::addParameter); - } - org.opensearch.client.Response response = client.getLowLevelClient().performRequest(request); - try (org.opensearch.core.xcontent.XContentParser parser = - org.opensearch.common.xcontent.json.JsonXContent.jsonXContent.createParser( - org.opensearch.core.xcontent.NamedXContentRegistry.EMPTY, - org.opensearch.common.xcontent.LoggingDeprecationHandler.INSTANCE, - response.getEntity().getContent())) { - return parser.map(); - } - } catch (IOException e) { - throw new IllegalStateException("Failed GET " + path, e); - } - } - - private static Long asLong(Object value) { - if (value == null) { - return null; - } - try { - return (long) Double.parseDouble(value.toString().trim()); - } catch (NumberFormatException e) { - return null; - } - } - - @Override - @SuppressWarnings("unchecked") - public List> resolveIndex(Map params) { - String expandWildcards = params == null ? null : params.get("expand_wildcards"); - Map body = - getJsonMap( - "/_resolve/index/*", - expandWildcards == null ? Map.of() : Map.of("expand_wildcards", expandWildcards)); - List> rows = new ArrayList<>(); - addResolved(body.get("indices"), "index", rows); - addResolved(body.get("aliases"), "alias", rows); - addResolved(body.get("data_streams"), "data_stream", rows); - return rows; - } - - @SuppressWarnings("unchecked") - private void addResolved(Object section, String type, List> rows) { - if (section instanceof List) { - for (Object o : (List) section) { - if (o instanceof Map) { - Map row = new HashMap<>(); - row.put("name", ((Map) o).get("name")); - row.put("type", type); - rows.add(row); - } - } - } - } - - private Map flattenHealth(ClusterHealthResponse response) { - Map row = new HashMap<>(); - row.put("cluster_name", response.getClusterName()); - row.put("status", response.getStatus().name().toLowerCase(Locale.ROOT)); - row.put("number_of_nodes", response.getNumberOfNodes()); - row.put("number_of_data_nodes", response.getNumberOfDataNodes()); - row.put("active_primary_shards", response.getActivePrimaryShards()); - row.put("active_shards", response.getActiveShards()); - row.put("relocating_shards", response.getRelocatingShards()); - row.put("initializing_shards", response.getInitializingShards()); - row.put("unassigned_shards", response.getUnassignedShards()); - row.put("timed_out", response.isTimedOut()); - return row; - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java deleted file mode 100644 index 9905ca564ab..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableCatalogScanRule.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.planner.rules; - -import org.apache.calcite.adapter.enumerable.EnumerableConvention; -import org.apache.calcite.plan.Convention; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableCatalogScan; -import org.opensearch.sql.opensearch.storage.system.CalciteLogicalCatalogScan; -import org.opensearch.sql.opensearch.storage.system.CalciteScannableCatalogScan; - -/** - * Rule to convert a {@link CalciteLogicalCatalogScan} into an enumerable scan: a {@link - * CalciteScannableCatalogScan} when the source opts into {@code Scannable}, otherwise a plain - * {@link CalciteEnumerableCatalogScan}. - */ -public class EnumerableCatalogScanRule extends ConverterRule { - /** Default configuration. */ - public static final Config DEFAULT_CONFIG = - Config.INSTANCE - .as(Config.class) - .withConversion( - CalciteLogicalCatalogScan.class, - s -> s.getCatalogTable() != null, - Convention.NONE, - EnumerableConvention.INSTANCE, - "EnumerableCatalogScanRule") - .withRuleFactory(EnumerableCatalogScanRule::new); - - protected EnumerableCatalogScanRule(Config config) { - super(config); - } - - @Override - public boolean matches(RelOptRuleCall call) { - CalciteLogicalCatalogScan scan = call.rel(0); - return scan.getVariablesSet().isEmpty(); - } - - @Override - public RelNode convert(RelNode rel) { - final CalciteLogicalCatalogScan scan = (CalciteLogicalCatalogScan) rel; - if (scan.getCatalogTable().getSource().isScannable()) { - return new CalciteScannableCatalogScan( - scan.getCluster(), - scan.getHints(), - scan.getTable(), - scan.getCatalogTable(), - scan.getSchema()); - } - return new CalciteEnumerableCatalogScan( - scan.getCluster(), - scan.getHints(), - scan.getTable(), - scan.getCatalogTable(), - scan.getSchema()); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java new file mode 100644 index 00000000000..616d1178873 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/EnumerableSystemIndexScanRule.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.planner.rules; + +import org.apache.calcite.adapter.enumerable.EnumerableConvention; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.opensearch.sql.opensearch.storage.system.CalciteEnumerableSystemIndexScan; +import org.opensearch.sql.opensearch.storage.system.CalciteLogicalSystemIndexScan; + +/** + * Rule to convert a {@link CalciteLogicalSystemIndexScan} to a {@link + * CalciteEnumerableSystemIndexScan}. + */ +public class EnumerableSystemIndexScanRule extends ConverterRule { + /** Default configuration. */ + public static final Config DEFAULT_CONFIG = + Config.INSTANCE + .as(Config.class) + .withConversion( + CalciteLogicalSystemIndexScan.class, + s -> s.getSysIndex() != null, + Convention.NONE, + EnumerableConvention.INSTANCE, + "EnumerableSystemIndexScanRule") + .withRuleFactory(EnumerableSystemIndexScanRule::new); + + /** Creates an EnumerableProjectRule. */ + protected EnumerableSystemIndexScanRule(Config config) { + super(config); + } + + @Override + public boolean matches(RelOptRuleCall call) { + CalciteLogicalSystemIndexScan scan = call.rel(0); + return scan.getVariablesSet().isEmpty(); + } + + @Override + public RelNode convert(RelNode rel) { + final CalciteLogicalSystemIndexScan scan = (CalciteLogicalSystemIndexScan) rel; + return new CalciteEnumerableSystemIndexScan( + scan.getCluster(), scan.getHints(), scan.getTable(), scan.getSysIndex(), scan.getSchema()); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java index c200ffa8909..3c8508cc455 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/OpenSearchIndexRules.java @@ -12,8 +12,8 @@ public class OpenSearchIndexRules { private static final RelOptRule INDEX_SCAN_RULE = EnumerableIndexScanRule.DEFAULT_CONFIG.toRule(); - private static final RelOptRule CATALOG_SCAN_RULE = - EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule(); + private static final RelOptRule SYSTEM_INDEX_SCAN_RULE = + EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule NESTED_AGGREGATE_RULE = EnumerableNestedAggregateRule.DEFAULT_CONFIG.toRule(); private static final RelOptRule GRAPH_LOOKUP_RULE = @@ -26,7 +26,7 @@ public class OpenSearchIndexRules { public static final List OPEN_SEARCH_NON_PUSHDOWN_RULES = ImmutableList.of( INDEX_SCAN_RULE, - CATALOG_SCAN_RULE, + SYSTEM_INDEX_SCAN_RULE, NESTED_AGGREGATE_RULE, GRAPH_LOOKUP_RULE, RELEVANCE_FUNCTION_RULE); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index ce2bdd4960b..b596c7bc47a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -71,17 +71,6 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); - public static final Setting PPL_REST_REDACTION_ENABLED_SETTING = - Setting.boolSetting( - Key.PPL_REST_REDACTION_ENABLED.getKeyValue(), false, Setting.Property.NodeScope); - - public static final Setting> PPL_REST_ALLOWED_ENDPOINTS_SETTING = - Setting.listSetting( - Key.PPL_REST_ALLOWED_ENDPOINTS.getKeyValue(), - List.of(), - Function.identity(), - Setting.Property.NodeScope); - public static final Setting PPL_QUERY_TIMEOUT_SETTING = Setting.positiveTimeSetting( Key.PPL_QUERY_TIMEOUT.getKeyValue(), @@ -391,16 +380,6 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_ENABLED, PPL_ENABLED_SETTING, new Updater(Key.PPL_ENABLED)); - registerNonDynamicSettings( - settingBuilder, - clusterSettings, - Key.PPL_REST_REDACTION_ENABLED, - PPL_REST_REDACTION_ENABLED_SETTING); - registerNonDynamicSettings( - settingBuilder, - clusterSettings, - Key.PPL_REST_ALLOWED_ENDPOINTS, - PPL_REST_ALLOWED_ENDPOINTS_SETTING); register( settingBuilder, clusterSettings, @@ -661,9 +640,7 @@ private void registerNonDynamicSettings( Settings.Key key, Setting setting) { settingBuilder.put(key, setting); - if (clusterSettings.get(setting) != null) { - latestSettings.put(key, clusterSettings.get(setting)); - } + latestSettings.put(key, clusterSettings.get(setting)); } /** @@ -734,8 +711,6 @@ public static List> pluginNonDynamicSettings() { return new ImmutableList.Builder>() .add(DATASOURCE_MASTER_SECRET_KEY) .add(DATASOURCE_CONFIG) - .add(PPL_REST_REDACTION_ENABLED_SETTING) - .add(PPL_REST_ALLOWED_ENDPOINTS_SETTING) .build(); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java index 7b911471242..1b7de315fb6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngine.java @@ -5,8 +5,6 @@ package org.opensearch.sql.opensearch.storage; -import static org.opensearch.sql.utils.SystemIndexUtils.decodeRestSpec; -import static org.opensearch.sql.utils.SystemIndexUtils.isRestSource; import static org.opensearch.sql.utils.SystemIndexUtils.isSystemIndex; import java.util.Collection; @@ -17,13 +15,9 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.rest.RestCatalogSource; -import org.opensearch.sql.opensearch.storage.rest.RestEndpointRegistry; -import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; -import org.opensearch.sql.opensearch.storage.system.SystemIndexCatalogSource; +import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; /** OpenSearch storage engine implementation. */ @RequiredArgsConstructor @@ -41,29 +35,10 @@ public Collection getFunctions() { @Override public Table getTable(DataSourceSchemaName dataSourceSchemaName, String name) { - if (isRestSource(name)) { - return restTable(name); - } else if (isSystemIndex(name)) { - return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); + if (isSystemIndex(name)) { + return new OpenSearchSystemIndex(client, settings, name); } else { return new OpenSearchIndex(client, settings, name); } } - - private Table restTable(String name) { - RestSpec spec = decodeRestSpec(name); - RestEndpointRegistry.resolve(spec.getEndpoint()); - List allowed = settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS); - if (allowed == null || !(allowed.contains("*") || allowed.contains(spec.getEndpoint()))) { - throw new IllegalArgumentException( - allowed == null || allowed.isEmpty() - ? "the rest command is disabled on this cluster" - : "rest endpoint [" - + spec.getEndpoint() - + "] is not enabled on this cluster. Enabled endpoints: " - + allowed); - } - boolean redact = settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED); - return new OpenSearchCatalogTable(new RestCatalogSource(client, spec, redact), settings); - } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java deleted file mode 100644 index 96779726a7e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSource.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.Map; -import lombok.Getter; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.opensearch.storage.system.CatalogSource; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * {@link CatalogSource} for the {@code rest} command: an allow-listed, read-only management - * endpoint resolved against {@link RestEndpointRegistry}, exposing the fixed endpoint schema. - * Calcite only (no V2 path) and {@code Scannable} for the {@code collect} short-circuit. - */ -@Getter -public class RestCatalogSource implements CatalogSource { - - private final OpenSearchClient client; - private final RestSpec spec; - private final RestEndpointRegistry.Endpoint endpoint; - private final boolean redact; - - public RestCatalogSource(OpenSearchClient client, RestSpec spec) { - this(client, spec, false); - } - - public RestCatalogSource(OpenSearchClient client, RestSpec spec, boolean redact) { - this.client = client; - this.spec = spec; - this.redact = redact; - // Allow-list enforced here: unknown or mutating endpoints and disallowed args are rejected. - this.endpoint = RestEndpointRegistry.resolve(spec.getEndpoint()); - RestEndpointRegistry.validate(spec); - } - - @Override - public Map getFieldTypes() { - return endpoint.getSchema(); - } - - @Override - public OpenSearchSystemRequest createRequest() { - return new RestRequest(client, endpoint, spec, redact); - } - - @Override - public boolean isScannable() { - return true; - } - - @Override - public PhysicalPlan implementV2(LogicalPlan plan) { - throw new UnsupportedOperationException("rest command is supported only on the Calcite engine"); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java deleted file mode 100644 index e64a91ab54a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistry.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; -import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; -import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; -import static org.opensearch.sql.data.model.ExprValueUtils.longValue; -import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; -import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; -import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.LONG; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import lombok.Getter; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * The read-only endpoint allow-list expressed as data: each allow-listed, read-only endpoint maps - * to its transport action (a read-only call on {@link OpenSearchClient}), a fixed output schema (so - * the Calcite plan can fix its row type at plan time), and the query args it accepts. - * - *

This is the single place the read-only allow-list is enforced. Endpoints outside the registry, - * including every mutating endpoint, are rejected by {@link #resolve} with a clear exception. - * Adding an endpoint is a reviewed change here, never arbitrary pass-through. - */ -public final class RestEndpointRegistry { - - private RestEndpointRegistry() {} - - /** Produces the raw rows for an endpoint via a read-only client call. */ - @FunctionalInterface - public interface RowFetcher { - List> fetch(OpenSearchClient client, RestSpec spec); - } - - /** A single allow-listed endpoint description. */ - @Getter - public static final class Endpoint { - private final String path; - private final LinkedHashMap schema; - private final Set allowedArgs; - private final RowFetcher fetcher; - - Endpoint( - String path, - LinkedHashMap schema, - Set allowedArgs, - RowFetcher fetcher) { - this.path = path; - this.schema = schema; - this.allowedArgs = allowedArgs; - this.fetcher = fetcher; - } - - /** Dispatch the read-only call and shape the response into fixed-schema rows. */ - public List toRows(OpenSearchClient client, RestSpec spec) { - return toRows(client, spec, false); - } - - /** - * Shape the response into fixed-schema rows, masking network identifiers when redaction is - * enabled. {@code /_cat/*} cells are fully masked and the {@code /_cluster/settings} value - * column is zone-masked. Off by default. - */ - public List toRows(OpenSearchClient client, RestSpec spec, boolean redact) { - boolean redactCat = redact && path.startsWith("/_cat"); - boolean redactSettingsValue = redact && "/_cluster/settings".equals(path); - List out = new ArrayList<>(); - for (Map raw : fetcher.fetch(client, spec)) { - LinkedHashMap tuple = new LinkedHashMap<>(); - for (Map.Entry col : schema.entrySet()) { - ExprValue value = coerce(col.getKey(), col.getValue(), raw.get(col.getKey())); - tuple.put( - col.getKey(), - maskCell(col.getKey(), col.getValue(), value, redactCat, redactSettingsValue)); - } - out.add(new ExprTupleValue(tuple)); - } - return out; - } - - private static ExprValue maskCell( - String column, - ExprType type, - ExprValue value, - boolean redactCat, - boolean redactSettingsValue) { - if (type != STRING || value.isNull()) { - return value; - } - if (redactCat) { - return stringValue(RestResponseRedactor.redact(value.stringValue())); - } - if (redactSettingsValue && "value".equals(column)) { - return stringValue(RestResponseRedactor.maskAvailabilityZone(value.stringValue())); - } - return value; - } - } - - private static final Map REGISTRY = buildRegistry(); - - private static Map buildRegistry() { - Map m = new LinkedHashMap<>(); - - // /_cluster/health — single-row cluster health snapshot (read-only monitor action). - LinkedHashMap healthSchema = new LinkedHashMap<>(); - healthSchema.put("cluster_name", STRING); - healthSchema.put("status", STRING); - healthSchema.put("number_of_nodes", INTEGER); - healthSchema.put("number_of_data_nodes", INTEGER); - healthSchema.put("active_primary_shards", INTEGER); - healthSchema.put("active_shards", INTEGER); - healthSchema.put("relocating_shards", INTEGER); - healthSchema.put("initializing_shards", INTEGER); - healthSchema.put("unassigned_shards", INTEGER); - healthSchema.put("timed_out", BOOLEAN); - m.put( - "/_cluster/health", - new Endpoint( - "/_cluster/health", - healthSchema, - Set.of("local"), - (client, spec) -> List.of(client.clusterHealth(spec.getArgs())))); - - // /_cat/indices — one row per index (read-only monitor action). - LinkedHashMap catSchema = new LinkedHashMap<>(); - catSchema.put("index", STRING); - catSchema.put("health", STRING); - catSchema.put("pri", INTEGER); - catSchema.put("rep", INTEGER); - catSchema.put("active_shards", INTEGER); - m.put( - "/_cat/indices", - new Endpoint( - "/_cat/indices", - catSchema, - Set.of("health"), - (client, spec) -> client.catIndices(spec.getArgs()))); - - // /_cat/nodes — one row per node with resource state (read-only monitor action). - LinkedHashMap nodesSchema = new LinkedHashMap<>(); - nodesSchema.put("name", STRING); - nodesSchema.put("ip", STRING); - nodesSchema.put("node_role", STRING); - nodesSchema.put("heap_percent", INTEGER); - nodesSchema.put("ram_percent", INTEGER); - nodesSchema.put("cpu", INTEGER); - m.put( - "/_cat/nodes", - new Endpoint( - "/_cat/nodes", - nodesSchema, - Set.of(), - (client, spec) -> client.catNodes(spec.getArgs()))); - - // /_cat/cluster_manager — single row identifying the elected cluster manager node. - LinkedHashMap clusterManagerSchema = new LinkedHashMap<>(); - clusterManagerSchema.put("id", STRING); - clusterManagerSchema.put("host", STRING); - clusterManagerSchema.put("ip", STRING); - clusterManagerSchema.put("node", STRING); - m.put( - "/_cat/cluster_manager", - new Endpoint( - "/_cat/cluster_manager", - clusterManagerSchema, - Set.of(), - (client, spec) -> client.catClusterManager(spec.getArgs()))); - - // /_cat/plugins — one row per installed plugin per node (read-only monitor action). - LinkedHashMap pluginsSchema = new LinkedHashMap<>(); - pluginsSchema.put("name", STRING); - pluginsSchema.put("component", STRING); - pluginsSchema.put("version", STRING); - m.put( - "/_cat/plugins", - new Endpoint( - "/_cat/plugins", - pluginsSchema, - Set.of(), - (client, spec) -> client.catPlugins(spec.getArgs()))); - - // /_cat/shards — one row per shard (read-only monitor action). - LinkedHashMap shardsSchema = new LinkedHashMap<>(); - shardsSchema.put("index", STRING); - shardsSchema.put("shard", INTEGER); - shardsSchema.put("prirep", STRING); - shardsSchema.put("state", STRING); - shardsSchema.put("node", STRING); - m.put( - "/_cat/shards", - new Endpoint( - "/_cat/shards", - shardsSchema, - Set.of(), - (client, spec) -> client.catShards(spec.getArgs()))); - - // /_cluster/state — single-row cluster-state epoch (version, uuid, manager node). - LinkedHashMap stateSchema = new LinkedHashMap<>(); - stateSchema.put("cluster_name", STRING); - stateSchema.put("state_uuid", STRING); - stateSchema.put("version", LONG); - stateSchema.put("cluster_manager_node", STRING); - m.put( - "/_cluster/state", - new Endpoint( - "/_cluster/state", - stateSchema, - Set.of(), - (client, spec) -> List.of(client.clusterState(spec.getArgs())))); - - // /_cluster/settings — one row per configured setting (persistent/transient tier). - LinkedHashMap settingsSchema = new LinkedHashMap<>(); - settingsSchema.put("setting", STRING); - settingsSchema.put("value", STRING); - settingsSchema.put("tier", STRING); - m.put( - "/_cluster/settings", - new Endpoint( - "/_cluster/settings", - settingsSchema, - Set.of(), - (client, spec) -> client.clusterSettings(spec.getArgs()))); - - // /_resolve/index — one row per resolved index/alias/data_stream name. - LinkedHashMap resolveSchema = new LinkedHashMap<>(); - resolveSchema.put("name", STRING); - resolveSchema.put("type", STRING); - m.put( - "/_resolve/index", - new Endpoint( - "/_resolve/index", - resolveSchema, - Set.of("expand_wildcards"), - (client, spec) -> client.resolveIndex(spec.getArgs()))); - - return m; - } - - /** - * Resolve an allow-listed endpoint. Anything outside the registry (unknown path, mutating verb, - * {@code /services/*}, plugin admin endpoints) is refused here. - */ - public static Endpoint resolve(String path) { - if (path == null || path.isBlank()) { - throw new IllegalArgumentException( - "rest endpoint must be a non-empty path. Supported read-only endpoints: " - + REGISTRY.keySet()); - } - Endpoint endpoint = REGISTRY.get(path); - if (endpoint == null) { - throw new IllegalArgumentException( - "rest endpoint [" - + path - + "] is not allow-listed. Only read-only in-cluster endpoints are supported: " - + REGISTRY.keySet()); - } - return endpoint; - } - - /** Validate that every supplied query arg is accepted by the endpoint. */ - public static void validate(RestSpec spec) { - Endpoint endpoint = resolve(spec.getEndpoint()); - if (spec.getCount() != null && spec.getCount() < 0) { - throw new IllegalArgumentException( - "rest endpoint [" - + spec.getEndpoint() - + "] count must be a non-negative integer, got [" - + spec.getCount() - + "]"); - } - if (spec.getTimeout() != null) { - // The timeout token is reserved in the grammar for forward compatibility, but a single - // uniform timeout cannot map cleanly across the endpoints (wait-for-status vs - // cluster-manager vs client socket timeouts differ per action). Reject it with a clear - // client error rather than silently ignoring it. - throw new IllegalArgumentException( - "rest endpoint [" + spec.getEndpoint() + "] does not support the timeout argument yet"); - } - if (spec.getArgs() != null) { - for (String arg : spec.getArgs().keySet()) { - if (!endpoint.getAllowedArgs().contains(arg)) { - throw new IllegalArgumentException( - "rest endpoint [" - + spec.getEndpoint() - + "] does not accept arg [" - + arg - + "]. Allowed args: " - + endpoint.getAllowedArgs()); - } - validateArgValue(spec.getEndpoint(), arg, spec.getArgs().get(arg)); - } - } - } - - // Allowed value domains for the get-args that are applied server-side. Keys are validated against - // the per-endpoint allow-list above; values are validated here so a user-supplied value is never - // passed unchecked into an admin transport request. - private static final Map> ARG_VALUE_DOMAINS = - Map.of( - "local", Set.of("true", "false"), - "health", Set.of("green", "yellow", "red")); - - private static final Set EXPAND_WILDCARDS_VALUES = - Set.of("open", "closed", "hidden", "none", "all"); - - /** Reject any get-arg value outside its allow-listed domain with a clear client error. */ - private static void validateArgValue(String endpoint, String arg, String value) { - Set domain = ARG_VALUE_DOMAINS.get(arg); - if (domain != null) { - if (value == null || !domain.contains(value.toLowerCase(java.util.Locale.ROOT))) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [" - + arg - + "] has an unsupported value [" - + value - + "]. Allowed values: " - + domain); - } - } else if ("expand_wildcards".equals(arg)) { - if (value == null || value.isBlank()) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); - } - for (String token : value.toLowerCase(java.util.Locale.ROOT).split(",")) { - if (!EXPAND_WILDCARDS_VALUES.contains(token.trim())) { - throw new IllegalArgumentException( - "rest endpoint [" - + endpoint - + "] arg [expand_wildcards] has an unsupported value [" - + value - + "]. Allowed values: " - + EXPAND_WILDCARDS_VALUES); - } - } - } - } - - private static ExprValue coerce(String column, ExprType type, Object value) { - if (value == null) { - return ExprNullValue.of(); - } - try { - if (type == INTEGER) { - return integerValue(toNumber(value).intValue()); - } - if (type == LONG) { - return longValue(toNumber(value).longValue()); - } - if (type == DOUBLE) { - return doubleValue(toNumber(value).doubleValue()); - } - if (type == BOOLEAN) { - return booleanValue(toBoolean(value)); - } - } catch (IllegalArgumentException | ClassCastException e) { - // Surface a clear client error (HTTP 400) instead of a raw HTTP 500 when an endpoint - // returns an unexpected value shape. NumberFormatException extends IllegalArgumentException, - // so toNumber parse failures and toBoolean's "not a boolean" are both caught here; genuinely - // unexpected faults (NPE, etc.) are left to propagate. - throw new IllegalArgumentException( - "rest endpoint value for column [" - + column - + "] could not be coerced to " - + type - + ": [" - + value - + "]"); - } - return stringValue(String.valueOf(value)); - } - - /** Coerce a transport/JSON value to a Number, parsing numeric strings (e.g. the cat JSON API). */ - private static Number toNumber(Object value) { - if (value instanceof Number n) { - return n; - } - String s = String.valueOf(value).trim(); - if (s.isEmpty()) { - throw new NumberFormatException("empty string"); - } - if (s.indexOf('.') >= 0 || s.indexOf('e') >= 0 || s.indexOf('E') >= 0) { - return Double.parseDouble(s); - } - return Long.parseLong(s); - } - - /** Coerce a transport/JSON value to a boolean, accepting Boolean or the strings true/false. */ - private static boolean toBoolean(Object value) { - if (value instanceof Boolean b) { - return b; - } - String s = String.valueOf(value).trim(); - if (s.isEmpty()) { - throw new IllegalArgumentException("empty string is not a boolean"); - } - if (s.equalsIgnoreCase("true")) { - return true; - } - if (s.equalsIgnoreCase("false")) { - return false; - } - throw new IllegalArgumentException("not a boolean: " + value); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java deleted file mode 100644 index 868dabdd64e..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestRequest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.List; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * Dispatches an allow-listed, read-only management endpoint through the transport node client under - * the caller's security thread-context and returns the response shaped to the endpoint's fixed - * schema. The {@code rest} analogue of {@code OpenSearchCatIndicesRequest}; it implements {@link - * OpenSearchSystemRequest} so the enumerator pattern (resource-monitored iteration) is identical to - * the system-index scan family. - */ -public class RestRequest implements OpenSearchSystemRequest { - - private final OpenSearchClient client; - private final RestEndpointRegistry.Endpoint endpoint; - private final RestSpec spec; - private final boolean redact; - - public RestRequest( - OpenSearchClient client, RestEndpointRegistry.Endpoint endpoint, RestSpec spec) { - this(client, endpoint, spec, false); - } - - public RestRequest( - OpenSearchClient client, - RestEndpointRegistry.Endpoint endpoint, - RestSpec spec, - boolean redact) { - this.client = client; - this.endpoint = endpoint; - this.spec = spec; - this.redact = redact; - } - - @Override - public List search() { - List rows = endpoint.toRows(client, spec, redact); - if (spec.getCount() != null && spec.getCount() >= 0 && rows.size() > spec.getCount()) { - return rows.subList(0, spec.getCount()); - } - return rows; - } - - @Override - public String toString() { - return "RestRequest{endpoint=" + endpoint.getPath() + "}"; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java deleted file mode 100644 index fd674dccde4..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactor.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import java.util.List; -import java.util.regex.Pattern; - -/** - * Masks network identifiers in rest command cell values. Enabled per deployment via {@code - * plugins.ppl.rest.redaction.enabled}; off by default. - */ -public final class RestResponseRedactor { - - private RestResponseRedactor() {} - - private static final String OCTET = "(25[0-5]|2[0-4]\\d|[0-1]?\\d\\d?)"; - private static final Pattern IPV4 = - Pattern.compile("\\b" + OCTET + "\\." + OCTET + "\\." + OCTET + "\\." + OCTET + "\\b"); - private static final Pattern INET = Pattern.compile("inet\\[/[\\d.:]+\\]"); - private static final Pattern EC2_HOST = - Pattern.compile("\\bip-" + OCTET + "-" + OCTET + "-" + OCTET + "-" + OCTET + "\\b"); - private static final Pattern IPV6 = - Pattern.compile( - "([0-9a-f]{1,4}:){7}[0-9a-f]{1,4}" - + "|([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?::([0-9a-f]{1,4}(:[0-9a-f]{1,4})*)?", - Pattern.CASE_INSENSITIVE); - private static final Pattern AZ_NAME = - Pattern.compile( - "\\b[a-z]{2}(-(gov|iso[a-z]?))?-(central|(north|south)?(east|west)?)-\\d[a-z]\\b", - Pattern.CASE_INSENSITIVE); - - private record Mask(Pattern pattern, String replacement) {} - - private static final List MASKS = - List.of( - new Mask(IPV4, "x.x.x.x"), - new Mask(INET, "inet[/x.x.x.x:y]"), - new Mask(EC2_HOST, ""), - new Mask(IPV6, "x.x.x.x"), - new Mask(AZ_NAME, "xx-xxxxx-xx")); - - /** Mask IPv4, inet, EC2 host names, IPv6, and availability-zone names in the text. */ - public static String redact(String text) { - if (text == null || text.isEmpty()) { - return text; - } - String out = text; - for (Mask mask : MASKS) { - out = mask.pattern().matcher(out).replaceAll(mask.replacement()); - } - return out; - } - - /** Mask availability-zone names only. */ - public static String maskAvailabilityZone(String text) { - if (text == null || text.isEmpty()) { - return text; - } - return AZ_NAME.matcher(text).replaceAll("xx-xxxxx-xx"); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java deleted file mode 100644 index d425c361e6a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/rest/RestSettingsFilterHolder.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import org.opensearch.common.settings.SettingsFilter; - -/** - * Bridge for sharing the node-level {@link SettingsFilter} with the in-cluster {@code rest ' - * /_cluster/settings'} fetcher. - * - *

The native {@code GET /_cluster/settings} REST endpoint redacts settings registered with - * {@code Property.Filtered} (or matched by a plugin-registered filter pattern) by running the - * response through {@link SettingsFilter}. The PPL {@code rest} command's in-cluster path reads - * {@code persistentSettings()}/{@code transientSettings()} straight from cluster state via the - * transport layer, where no {@link SettingsFilter} is applied. To keep the command's redaction - * behavior identical to the native endpoint, the node's {@link SettingsFilter} is published here. - * - *

Why a static holder: the {@link SettingsFilter} instance is only handed to the plugin in - * {@code SQLPlugin#getRestHandlers}, which runs outside any Guice-managed lifecycle, while {@link - * OpenSearchNodeClient} is built through the Node injector. Persisting the filter here once {@code - * getRestHandlers} fires lets the fetcher read the same instance without going back through the - * injector. This mirrors the existing {@code AnalyticsExecutorHolder} pattern. - */ -public final class RestSettingsFilterHolder { - - private static volatile SettingsFilter settingsFilter; - - private RestSettingsFilterHolder() {} - - public static void set(SettingsFilter instance) { - settingsFilter = instance; - } - - public static SettingsFilter get() { - return settingsFilter; - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java similarity index 68% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java index 19d585ea684..fa543ab266b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteCatalogScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/AbstractCalciteSystemIndexScan.java @@ -16,21 +16,21 @@ import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -/** An abstract relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ +/** An abstract relational operator representing a scan of an OpenSearchSystemIndex type. */ @Getter -public abstract class AbstractCalciteCatalogScan extends TableScan { - public final OpenSearchCatalogTable catalogTable; +public abstract class AbstractCalciteSystemIndexScan extends TableScan { + public final OpenSearchSystemIndex sysIndex; protected final RelDataType schema; - protected AbstractCalciteCatalogScan( + protected AbstractCalciteSystemIndexScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchCatalogTable catalogTable, + OpenSearchSystemIndex sysIndex, RelDataType schema) { super(cluster, traitSet, hints, table); - this.catalogTable = requireNonNull(catalogTable, "OpenSearch catalog table"); + this.sysIndex = requireNonNull(sysIndex, "OpenSearch system index"); this.schema = schema; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java similarity index 80% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java index 58f86874c73..b0c92dce8f9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableCatalogScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteEnumerableSystemIndexScan.java @@ -26,22 +26,17 @@ import org.apache.calcite.util.Pair; import org.checkerframework.checker.nullness.qual.Nullable; -/** The physical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ -public class CalciteEnumerableCatalogScan extends AbstractCalciteCatalogScan +/** The physical relational operator representing a scan of an OpenSearchSystemIndex type. */ +public class CalciteEnumerableSystemIndexScan extends AbstractCalciteSystemIndexScan implements EnumerableRel { - public CalciteEnumerableCatalogScan( + public CalciteEnumerableSystemIndexScan( RelOptCluster cluster, List hints, RelOptTable table, - OpenSearchCatalogTable catalogTable, + OpenSearchSystemIndex sysIndex, RelDataType schema) { super( - cluster, - cluster.traitSetOf(EnumerableConvention.INSTANCE), - hints, - table, - catalogTable, - schema); + cluster, cluster.traitSetOf(EnumerableConvention.INSTANCE), hints, table, sysIndex, schema); } @Override @@ -65,7 +60,7 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(), pref.preferArray()); - Expression scanOperator = implementor.stash(this, CalciteEnumerableCatalogScan.class); + Expression scanOperator = implementor.stash(this, CalciteEnumerableSystemIndexScan.class); return implementor.result(physType, Blocks.toBlock(Expressions.call(scanOperator, "scan"))); } @@ -73,10 +68,10 @@ public Result implement(EnumerableRelImplementor implementor, Prefer pref) { return new AbstractEnumerable<>() { @Override public Enumerator enumerator() { - return new OpenSearchCatalogEnumerator( + return new OpenSearchSystemIndexEnumerator( getFieldPath(), - catalogTable.getSource().createRequest(), - catalogTable.createOpenSearchResourceMonitor()); + sysIndex.getSystemIndexBundle().getRight(), + sysIndex.createOpenSearchResourceMonitor()); } }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java similarity index 59% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java index 4278539c2b0..012ceec8c13 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalCatalogScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteLogicalSystemIndexScan.java @@ -14,35 +14,35 @@ import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.opensearch.planner.rules.EnumerableCatalogScanRule; +import org.opensearch.sql.opensearch.planner.rules.EnumerableSystemIndexScanRule; -/** The logical relational operator representing a scan of an {@link OpenSearchCatalogTable}. */ -public class CalciteLogicalCatalogScan extends AbstractCalciteCatalogScan { +/** The logical relational operator representing a scan of an OpenSearchSystemIndex type. */ +public class CalciteLogicalSystemIndexScan extends AbstractCalciteSystemIndexScan { - public CalciteLogicalCatalogScan( - RelOptCluster cluster, RelOptTable table, OpenSearchCatalogTable catalogTable) { + public CalciteLogicalSystemIndexScan( + RelOptCluster cluster, RelOptTable table, OpenSearchSystemIndex sysIndex) { this( cluster, cluster.traitSetOf(Convention.NONE), ImmutableList.of(), table, - catalogTable, + sysIndex, table.getRowType()); } - protected CalciteLogicalCatalogScan( + protected CalciteLogicalSystemIndexScan( RelOptCluster cluster, RelTraitSet traitSet, List hints, RelOptTable table, - OpenSearchCatalogTable catalogTable, + OpenSearchSystemIndex sysIndex, RelDataType schema) { - super(cluster, traitSet, hints, table, catalogTable, schema); + super(cluster, traitSet, hints, table, sysIndex, schema); } @Override public void register(RelOptPlanner planner) { super.register(planner); - planner.addRule(EnumerableCatalogScanRule.DEFAULT_CONFIG.toRule()); + planner.addRule(EnumerableSystemIndexScanRule.DEFAULT_CONFIG.toRule()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java deleted file mode 100644 index 1d021cda354..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CalciteScannableCatalogScan.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.List; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rel.type.RelDataType; -import org.opensearch.sql.calcite.plan.Scannable; - -/** - * A {@link CalciteEnumerableCatalogScan} that additionally carries the {@link Scannable} marker, - * enabling the {@code collect} short-circuit. Produced when the {@link CatalogSource} opts in via - * {@link CatalogSource#isScannable()}. - */ -public class CalciteScannableCatalogScan extends CalciteEnumerableCatalogScan implements Scannable { - public CalciteScannableCatalogScan( - RelOptCluster cluster, - List hints, - RelOptTable table, - OpenSearchCatalogTable catalogTable, - RelDataType schema) { - super(cluster, hints, table, catalogTable, schema); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java deleted file mode 100644 index 85dabe06b9a..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/CatalogSource.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.Map; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; - -/** - * Strategy supplying the schema and row source for an {@link OpenSearchCatalogTable} backed by a - * read-only OpenSearch admin API. Each endpoint family plugs in its own {@code CatalogSource} - * rather than defining a separate table and Calcite scan hierarchy. - */ -public interface CatalogSource { - - /** Fixed schema mapping each column name to its type. */ - Map getFieldTypes(); - - /** - * Builds the read-only request whose {@link OpenSearchSystemRequest#search()} yields the rows. - */ - OpenSearchSystemRequest createRequest(); - - /** - * Whether the enumerable scan should carry the {@link org.opensearch.sql.calcite.plan.Scannable} - * marker for the {@code collect} short-circuit. Defaults to {@code false}. - */ - default boolean isScannable() { - return false; - } - - /** The V2 physical plan path, or throws when the source is Calcite only. */ - PhysicalPlan implementV2(LogicalPlan plan); -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java deleted file mode 100644 index 9bcb3a3ff0c..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTable.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import java.util.Map; -import lombok.Getter; -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.rel.RelNode; -import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; -import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.physical.PhysicalPlan; - -/** - * A single generic Calcite and V2 table over a read-only OpenSearch catalog endpoint. Per-endpoint - * behavior (schema, row source, V2 support, and the {@code Scannable} marker) is supplied by a - * pluggable {@link CatalogSource}. - */ -@Getter -public class OpenSearchCatalogTable extends AbstractOpenSearchTable { - - private final CatalogSource source; - private final Settings settings; - - public OpenSearchCatalogTable(CatalogSource source, Settings settings) { - this.source = source; - this.settings = settings; - } - - @Override - public boolean exists() { - return true; - } - - @Override - public void create(Map schema) { - throw new UnsupportedOperationException( - "OpenSearch catalog table is predefined and cannot be created"); - } - - @Override - public Map getFieldTypes() { - return source.getFieldTypes(); - } - - @Override - public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { - final RelOptCluster cluster = context.getCluster(); - return new CalciteLogicalCatalogScan(cluster, relOptTable, this); - } - - @Override - public PhysicalPlan implement(LogicalPlan plan) { - return source.implementV2(plan); - } - - public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { - return new OpenSearchResourceMonitor(settings, new OpenSearchMemoryHealthy(settings)); - } -} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java new file mode 100644 index 00000000000..6bae4d21aba --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndex.java @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.system; + +import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.calcite.plan.AbstractOpenSearchTable; +import org.opensearch.sql.common.setting.Settings; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.monitor.OpenSearchMemoryHealthy; +import org.opensearch.sql.opensearch.monitor.OpenSearchResourceMonitor; +import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; +import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; +import org.opensearch.sql.planner.DefaultImplementor; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalRelation; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.utils.SystemIndexUtils; + +/** OpenSearch System Index Table Implementation. */ +@Getter +public class OpenSearchSystemIndex extends AbstractOpenSearchTable { + /** System Index Name. */ + private final Pair systemIndexBundle; + + @Getter private final Settings settings; + + public OpenSearchSystemIndex(OpenSearchClient client, Settings settings, String indexName) { + this.systemIndexBundle = buildIndexBundle(client, indexName); + this.settings = settings; + } + + @Override + public boolean exists() { + return true; // TODO: implement for system index later + } + + @Override + public void create(Map schema) { + throw new UnsupportedOperationException( + "OpenSearch system index is predefined and cannot be created"); + } + + @Override + public Map getFieldTypes() { + return systemIndexBundle.getLeft().getMapping(); + } + + @Override + public RelNode toRel(RelOptTable.ToRelContext context, RelOptTable relOptTable) { + final RelOptCluster cluster = context.getCluster(); + return new CalciteLogicalSystemIndexScan(cluster, relOptTable, this); + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return plan.accept(new OpenSearchSystemIndexDefaultImplementor(), null); + } + + public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { + return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); + } + + @VisibleForTesting + @RequiredArgsConstructor + public class OpenSearchSystemIndexDefaultImplementor extends DefaultImplementor { + + @Override + public PhysicalPlan visitRelation(LogicalRelation node, Object context) { + return new OpenSearchSystemIndexScan(systemIndexBundle.getRight()); + } + } + + /** + * Constructor of ElasticsearchSystemIndexName. + * + * @param indexName index name; + */ + private Pair buildIndexBundle( + OpenSearchClient client, String indexName) { + SystemIndexUtils.SystemTable systemTable = systemTable(indexName); + if (systemTable.isSystemInfoTable()) { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); + } else { + return Pair.of( + OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, + new OpenSearchDescribeIndexRequest( + client, systemTable.getTableName(), systemTable.getLangSpec())); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java similarity index 90% rename from opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java rename to opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java index dff9b47265a..1bb8f9d5293 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexEnumerator.java @@ -16,11 +16,8 @@ import org.opensearch.sql.monitor.ResourceMonitor; import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -/** - * Resource-monitored iteration over the rows produced by a read-only catalog {@link - * OpenSearchSystemRequest}. - */ -public class OpenSearchCatalogEnumerator implements Enumerator { +/** Supports a simple iteration over a collection for OpenSearch system index */ +public class OpenSearchSystemIndexEnumerator implements Enumerator { /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -38,7 +35,7 @@ public class OpenSearchCatalogEnumerator implements Enumerator { /** ResourceMonitor. */ private final ResourceMonitor monitor; - public OpenSearchCatalogEnumerator( + public OpenSearchSystemIndexEnumerator( List fields, OpenSearchSystemRequest request, ResourceMonitor monitor) { this.fields = fields; this.request = request; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java deleted file mode 100644 index 3541e745e9d..00000000000 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/system/SystemIndexCatalogSource.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.system; - -import static org.opensearch.sql.utils.SystemIndexUtils.systemTable; - -import java.util.Map; -import org.apache.commons.lang3.tuple.Pair; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.request.system.OpenSearchCatIndicesRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest; -import org.opensearch.sql.opensearch.request.system.OpenSearchSystemRequest; -import org.opensearch.sql.planner.DefaultImplementor; -import org.opensearch.sql.planner.logical.LogicalPlan; -import org.opensearch.sql.planner.logical.LogicalRelation; -import org.opensearch.sql.planner.physical.PhysicalPlan; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** - * {@link CatalogSource} for the SHOW TABLES and DESCRIBE system tables, backed by index listing and - * index field mappings. - */ -public class SystemIndexCatalogSource implements CatalogSource { - - private final Pair bundle; - - public SystemIndexCatalogSource(OpenSearchClient client, String indexName) { - this.bundle = buildBundle(client, indexName); - } - - @Override - public Map getFieldTypes() { - return bundle.getLeft().getMapping(); - } - - @Override - public OpenSearchSystemRequest createRequest() { - return bundle.getRight(); - } - - @Override - public PhysicalPlan implementV2(LogicalPlan plan) { - return plan.accept( - new DefaultImplementor() { - @Override - public PhysicalPlan visitRelation(LogicalRelation node, Object context) { - return new OpenSearchSystemIndexScan(bundle.getRight()); - } - }, - null); - } - - private static Pair buildBundle( - OpenSearchClient client, String indexName) { - SystemIndexUtils.SystemTable systemTable = systemTable(indexName); - if (systemTable.isSystemInfoTable()) { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_TABLES, new OpenSearchCatIndicesRequest(client)); - } else { - return Pair.of( - OpenSearchSystemIndexSchema.SYS_TABLE_MAPPINGS, - new OpenSearchDescribeIndexRequest( - client, systemTable.getTableName(), systemTable.getLangSpec())); - } - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java deleted file mode 100644 index 33be6cc8482..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientClusterSettingsFilterTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.client; - -import static java.util.stream.Collectors.toSet; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Answers.RETURNS_DEEP_STUBS; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.opensearch.action.admin.cluster.state.ClusterStateRequest; -import org.opensearch.action.admin.cluster.state.ClusterStateResponse; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.settings.SettingsFilter; -import org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder; -import org.opensearch.transport.client.node.NodeClient; - -/** - * Verifies the in-cluster {@code rest '/_cluster/settings'} fetcher redacts filtered settings using - * the node {@link SettingsFilter}, matching the native {@code GET /_cluster/settings} endpoint. - */ -class OpenSearchNodeClientClusterSettingsFilterTest { - - @AfterEach - void clearHolder() { - RestSettingsFilterHolder.set(null); - } - - private OpenSearchNodeClient clientReturning(Settings persistent, Settings transientSettings) { - NodeClient nodeClient = mock(NodeClient.class, RETURNS_DEEP_STUBS); - ClusterStateResponse stateResp = mock(ClusterStateResponse.class, RETURNS_DEEP_STUBS); - when(nodeClient.admin().cluster().state(any(ClusterStateRequest.class)).actionGet()) - .thenReturn(stateResp); - when(stateResp.getState().metadata().persistentSettings()).thenReturn(persistent); - when(stateResp.getState().metadata().transientSettings()).thenReturn(transientSettings); - return new OpenSearchNodeClient(nodeClient); - } - - @Test - void clusterSettingsRedactsFilteredKeyWhenFilterPublished() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("plugins.secret.token", "supersecret") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Publish a filter that redacts the secret key, exactly as the native endpoint would. - RestSettingsFilterHolder.set(new SettingsFilter(List.of("plugins.secret.token"))); - - List> rows = client.clusterSettings(Map.of()); - Set keys = rows.stream().map(r -> (String) r.get("setting")).collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable"), "non-secret setting kept"); - assertFalse(keys.contains("plugins.secret.token"), "filtered setting must be redacted"); - } - - @Test - void clusterSettingsRedactsByGlobPattern() { - Settings persistent = - Settings.builder() - .put("cluster.routing.allocation.enable", "all") - .put("s3.client.default.secret_key", "AKIAEXAMPLE") - .build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - RestSettingsFilterHolder.set(new SettingsFilter(List.of("s3.client.*.secret_key"))); - - Set keys = - client.clusterSettings(Map.of()).stream() - .map(r -> (String) r.get("setting")) - .collect(toSet()); - - assertTrue(keys.contains("cluster.routing.allocation.enable")); - assertFalse(keys.contains("s3.client.default.secret_key"), "glob-matched secret redacted"); - } - - @Test - void clusterSettingsFailsClosedWhenNoFilterPublished() { - Settings persistent = Settings.builder().put("plugins.secret.token", "supersecret").build(); - OpenSearchNodeClient client = clientReturning(persistent, Settings.EMPTY); - - // Fail closed: without a published SettingsFilter the command must refuse rather than leak raw - // (potentially secret-bearing) settings. At runtime getRestHandlers always publishes the filter - // before any query, so this path is unreachable in cluster. - assertThrows(IllegalStateException.class, () -> client.clusterSettings(Map.of())); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 0c570098924..5024d416086 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -9,15 +9,12 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.AdditionalMatchers.not; import static org.mockito.AdditionalMatchers.or; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL_SETTING; -import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_ALLOWED_ENDPOINTS_SETTING; -import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.PPL_REST_REDACTION_ENABLED_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.QUERY_MEMORY_LIMIT_SETTING; import static org.opensearch.sql.opensearch.setting.OpenSearchSettings.SPARK_EXECUTION_ENGINE_CONFIG; @@ -77,15 +74,6 @@ void pluginNonDynamicSettings() { assertFalse(settings.isEmpty()); } - @Test - void restSettingsAreNonDynamic() { - assertFalse(PPL_REST_REDACTION_ENABLED_SETTING.isDynamic()); - assertFalse(PPL_REST_ALLOWED_ENDPOINTS_SETTING.isDynamic()); - List> nonDynamic = OpenSearchSettings.pluginNonDynamicSettings(); - assertTrue(nonDynamic.contains(PPL_REST_REDACTION_ENABLED_SETTING)); - assertTrue(nonDynamic.contains(PPL_REST_ALLOWED_ENDPOINTS_SETTING)); - } - @Test void getSettings() { when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java index 102ec4da8f7..fa04395e065 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/OpenSearchStorageEngineTest.java @@ -7,15 +7,11 @@ import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.when; import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; import static org.opensearch.sql.utils.SystemIndexUtils.TABLE_INFO; import java.util.Collection; -import java.util.List; -import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -24,9 +20,8 @@ import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.expression.function.FunctionResolver; import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.opensearch.storage.system.OpenSearchCatalogTable; +import org.opensearch.sql.opensearch.storage.system.OpenSearchSystemIndex; import org.opensearch.sql.storage.Table; -import org.opensearch.sql.utils.SystemIndexUtils; @ExtendWith(MockitoExtension.class) class OpenSearchStorageEngineTest { @@ -57,68 +52,6 @@ public void getSystemTable() { OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); Table table = engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), TABLE_INFO); - assertAll( - () -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchCatalogTable)); - } - - @Test - public void getRestTableAllowedByWildcard() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("*")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); - Table table = - engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name); - assertTrue(table instanceof OpenSearchCatalogTable); - } - - @Test - public void getRestTableAllowedBySubset() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); - when(settings.getSettingValue(Settings.Key.PPL_REST_REDACTION_ENABLED)).thenReturn(false); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); - assertTrue( - engine.getTable(new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name) - instanceof OpenSearchCatalogTable); - } - - @Test - public void getRestTableRejectedWhenEndpointNotInSubset() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)) - .thenReturn(List.of("/_cat/nodes")); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cluster/settings", Map.of(), null, null)); - IllegalArgumentException e = - assertThrows( - IllegalArgumentException.class, - () -> - engine.getTable( - new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); - assertTrue(e.getMessage().contains("is not enabled on this cluster")); - } - - @Test - public void getRestTableDisabledWhenListEmpty() { - when(settings.getSettingValue(Settings.Key.PPL_REST_ALLOWED_ENDPOINTS)).thenReturn(List.of()); - OpenSearchStorageEngine engine = new OpenSearchStorageEngine(client, settings); - String name = - SystemIndexUtils.restTable( - new SystemIndexUtils.RestSpec("/_cat/nodes", Map.of(), null, null)); - IllegalArgumentException e = - assertThrows( - IllegalArgumentException.class, - () -> - engine.getTable( - new DataSourceSchemaName(DEFAULT_DATASOURCE_NAME, "default"), name)); - assertTrue(e.getMessage().contains("disabled on this cluster")); + assertAll(() -> assertNotNull(table), () -> assertTrue(table instanceof OpenSearchSystemIndex)); } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java deleted file mode 100644 index 8676373d2a6..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestCatalogSourceTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasEntry; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.type.ExprType; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -/** - * Covers the {@code rest} {@link RestCatalogSource}: fixed endpoint schema, allow-list enforcement, - * response row shaping and truncation, the {@code Scannable} opt-in, and the Calcite only (no V2) - * path. - */ -@ExtendWith(MockitoExtension.class) -class RestCatalogSourceTest { - - @Mock private OpenSearchClient client; - - private RestSpec healthSpec() { - return new RestSpec("/_cluster/health", Map.of(), null, null); - } - - @Test - void getFieldTypesReturnsFixedEndpointSchema() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); - Map fieldTypes = source.getFieldTypes(); - assertThat(fieldTypes, hasEntry("status", STRING)); - assertThat(fieldTypes, hasEntry("number_of_nodes", INTEGER)); - } - - @Test - void isScannable() { - assertTrue(new RestCatalogSource(client, healthSpec()).isScannable()); - } - - @Test - void implementV2IsUnsupported() { - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); - assertThrows(UnsupportedOperationException.class, () -> source.implementV2(null)); - } - - @Test - void constructorRejectsNonAllowListedEndpoint() { - assertThrows( - IllegalArgumentException.class, - () -> - new RestCatalogSource(client, new RestSpec("/_cluster/reroute", Map.of(), null, null))); - } - - @Test - void constructorRejectsDisallowedArg() { - assertThrows( - IllegalArgumentException.class, - () -> - new RestCatalogSource( - client, new RestSpec("/_cluster/health", Map.of("bad", "x"), null, null))); - } - - @Test - void constructorRejectsNegativeCount() { - assertThrows( - IllegalArgumentException.class, - () -> new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), -1, null))); - } - - @Test - void constructorRejectsTimeoutArg() { - assertThrows( - IllegalArgumentException.class, - () -> - new RestCatalogSource(client, new RestSpec("/_cluster/health", Map.of(), null, "5s"))); - } - - @Test - void restRequestShapesResponseRows() { - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestCatalogSource source = new RestCatalogSource(client, healthSpec()); - List rows = source.createRequest().search(); - assertEquals(1, rows.size()); - assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); - assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - } - - @Test - void countTruncatesRows() { - Map idx1 = new LinkedHashMap<>(); - idx1.put("index", "a"); - Map idx2 = new LinkedHashMap<>(); - idx2.put("index", "b"); - when(client.catIndices(any())).thenReturn(List.of(idx1, idx2)); - - RestCatalogSource source = - new RestCatalogSource(client, new RestSpec("/_cat/indices", Map.of(), 1, null)); - List rows = source.createRequest().search(); - assertEquals(1, rows.size()); - assertEquals("a", rows.get(0).tupleValue().get("index").stringValue()); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java deleted file mode 100644 index c5a978bb495..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestEndpointRegistryTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.when; -import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.opensearch.client.OpenSearchClient; -import org.opensearch.sql.utils.SystemIndexUtils.RestSpec; - -@ExtendWith(MockitoExtension.class) -class RestEndpointRegistryTest { - - @Mock private OpenSearchClient client; - - @Test - void resolveAllowListedEndpoint() { - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - assertEquals("/_cluster/health", endpoint.getPath()); - assertEquals(STRING, endpoint.getSchema().get("status")); - assertEquals(INTEGER, endpoint.getSchema().get("number_of_nodes")); - } - - @Test - void resolveRejectsNonAllowListedEndpoint() { - // A mutating endpoint is simply absent from the registry and is refused here. - assertThrows( - IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("/_cluster/reroute")); - assertThrows( - IllegalArgumentException.class, - () -> RestEndpointRegistry.resolve("/services/server/info")); - } - - @Test - void validateRejectsUnknownArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("not_allowed", "x"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - } - - @Test - void validateAcceptsAllowedArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of("local", "true"), null, null); - RestEndpointRegistry.validate(spec); // no throw - } - - @Test - void catEndpointsRedactAddressesWhenRedactionEnabled() { - Map node = new LinkedHashMap<>(); - node.put("name", "ip-10-0-0-7"); - node.put("ip", "10.0.0.7"); - node.put("node_role", "dir"); - node.put("heap_percent", 44); - node.put("ram_percent", 95); - node.put("cpu", 2); - when(client.catNodes(any())).thenReturn(List.of(node)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/nodes"); - RestSpec spec = new RestSpec("/_cat/nodes", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("", redacted.get("name").stringValue()); - assertEquals(44, redacted.get("heap_percent").integerValue()); - - Map plain = endpoint.toRows(client, spec, false).get(0).tupleValue(); - assertEquals("10.0.0.7", plain.get("ip").stringValue()); - assertEquals("ip-10-0-0-7", plain.get("name").stringValue()); - } - - @Test - void catClusterManagerRedactsHostAndIp() { - Map row = new LinkedHashMap<>(); - row.put("id", "fWhl6_ZQTaSJD9cJ82Ln2w"); - row.put("host", "10.0.0.7"); - row.put("ip", "10.0.0.7"); - row.put("node", "71d03b567bb755839a73d437b2b066d4"); - when(client.catClusterManager(any())).thenReturn(List.of(row)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/cluster_manager"); - RestSpec spec = new RestSpec("/_cat/cluster_manager", Map.of(), null, null); - - Map redacted = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("x.x.x.x", redacted.get("host").stringValue()); - assertEquals("x.x.x.x", redacted.get("ip").stringValue()); - assertEquals("fWhl6_ZQTaSJD9cJ82Ln2w", redacted.get("id").stringValue()); - assertEquals("71d03b567bb755839a73d437b2b066d4", redacted.get("node").stringValue()); - } - - @Test - void nonCatEndpointNotRedactedEvenWhenEnabled() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "10.0.0.7"); - health.put("status", "green"); - health.put("number_of_nodes", 3); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, null); - - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("10.0.0.7", row.get("cluster_name").stringValue()); - } - - @Test - void clusterSettingsMasksAvailabilityZoneInValue() { - Map setting = new LinkedHashMap<>(); - setting.put("setting", "cluster.routing.allocation.awareness.attributes"); - setting.put("value", "zone:us-east-1a"); - setting.put("tier", "persistent"); - when(client.clusterSettings(any())).thenReturn(List.of(setting)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/settings"); - RestSpec spec = new RestSpec("/_cluster/settings", Map.of(), null, null); - - Map row = endpoint.toRows(client, spec, true).get(0).tupleValue(); - assertEquals("zone:xx-xxxxx-xx", row.get("value").stringValue()); - assertEquals( - "cluster.routing.allocation.awareness.attributes", row.get("setting").stringValue()); - assertEquals("persistent", row.get("tier").stringValue()); - } - - @Test - void validateRejectsDroppedLevelArg() { - // level was dropped (no-op against the fixed cluster-level health schema); now unknown. - RestSpec spec = new RestSpec("/_cluster/health", Map.of("level", "indices"), null, null); - IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - assertTrue(ex.getMessage().contains("does not accept arg")); - } - - @Test - void validateRejectsDroppedFlatSettingsArg() { - // flat_settings was dropped (redundant: settings are already flattened to dotted keys). - RestSpec spec = new RestSpec("/_cluster/settings", Map.of("flat_settings", "true"), null, null); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - } - - @Test - void validateAcceptsValidArgValues() { - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "green"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open"), null, null)); - RestEndpointRegistry.validate( - new RestSpec("/_resolve/index", Map.of("expand_wildcards", "open,closed"), null, null)); - } - - @Test - void validateRejectsBadArgValue() { - IllegalArgumentException health = - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec("/_cat/indices", Map.of("health", "purple"), null, null))); - assertTrue(health.getMessage().contains("unsupported value")); - - IllegalArgumentException local = - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec("/_cluster/health", Map.of("local", "maybe"), null, null))); - assertTrue(local.getMessage().contains("unsupported value")); - - assertThrows( - IllegalArgumentException.class, - () -> - RestEndpointRegistry.validate( - new RestSpec( - "/_resolve/index", Map.of("expand_wildcards", "sideways"), null, null))); - } - - @Test - void resolveRejectsBlankEndpoint() { - IllegalArgumentException emptyEx = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve("")); - assertTrue(emptyEx.getMessage().contains("non-empty path")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(" ")); - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.resolve(null)); - } - - @Test - void validateRejectsNegativeCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), -1, null); - IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - assertTrue(ex.getMessage().contains("non-negative")); - } - - @Test - void validateAcceptsZeroCount() { - RestSpec spec = new RestSpec("/_cat/indices", Map.of(), 0, null); - RestEndpointRegistry.validate(spec); // no throw: 0 is a valid limit - } - - @Test - void validateRejectsTimeoutArg() { - RestSpec spec = new RestSpec("/_cluster/health", Map.of(), null, "5s"); - IllegalArgumentException ex = - assertThrows(IllegalArgumentException.class, () -> RestEndpointRegistry.validate(spec)); - assertTrue(ex.getMessage().contains("timeout")); - } - - @Test - void coerceParsesNumericStringValues() { - // The cat JSON API returns numeric columns as strings; coerce must parse them. - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "3"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - - assertEquals(3, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - } - - @Test - void coerceThrowsClearErrorOnUncoercibleValue() { - // A non-numeric value for an INTEGER column must surface a clear client error (HTTP 400), - // not a raw ClassCastException / NumberFormatException (HTTP 500). - Map health = new LinkedHashMap<>(); - health.put("status", "green"); - health.put("number_of_nodes", "not-a-number"); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - IllegalArgumentException ex = - assertThrows( - IllegalArgumentException.class, - () -> endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null))); - assertTrue(ex.getMessage().contains("number_of_nodes")); - assertTrue(ex.getMessage().contains("not-a-number")); - } - - @Test - void clusterHealthRowsAreShapedToFixedSchema() { - Map health = new LinkedHashMap<>(); - health.put("cluster_name", "test-cluster"); - health.put("status", "green"); - health.put("number_of_nodes", 1); - when(client.clusterHealth(any())).thenReturn(health); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cluster/health"); - List rows = - endpoint.toRows(client, new RestSpec("/_cluster/health", Map.of(), null, null)); - - assertEquals(1, rows.size()); - assertEquals("green", rows.get(0).tupleValue().get("status").stringValue()); - assertEquals(1, rows.get(0).tupleValue().get("number_of_nodes").integerValue()); - // a declared column the action did not return becomes null, never absent. - assertTrue(rows.get(0).tupleValue().get("relocating_shards").isNull()); - } - - @Test - void catIndicesRowsAreShapedToFixedSchema() { - Map idx = new LinkedHashMap<>(); - idx.put("index", "books"); - idx.put("health", "yellow"); - idx.put("pri", 1); - idx.put("rep", 1); - when(client.catIndices(any())).thenReturn(List.of(idx)); - - RestEndpointRegistry.Endpoint endpoint = RestEndpointRegistry.resolve("/_cat/indices"); - List rows = - endpoint.toRows(client, new RestSpec("/_cat/indices", Map.of(), null, null)); - - assertEquals(1, rows.size()); - assertEquals("books", rows.get(0).tupleValue().get("index").stringValue()); - assertEquals("yellow", rows.get(0).tupleValue().get("health").stringValue()); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java deleted file mode 100644 index 3b038306da5..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/rest/RestResponseRedactorTest.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.opensearch.storage.rest; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -class RestResponseRedactorTest { - - @Test - void masksIpv4() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("0.0.0.0")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("255.255.255.255")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("192.168.1.1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("1.2.3.4")); - assertEquals("x.x.x.x:9200", RestResponseRedactor.redact("10.0.0.1:9200")); - assertEquals("a x.x.x.x b x.x.x.x c", RestResponseRedactor.redact("a 10.0.0.1 b 172.16.5.4 c")); - } - - @Test - void doesNotMaskInvalidOrPartialIpv4() { - assertEquals("256.1.1.1", RestResponseRedactor.redact("256.1.1.1")); - assertEquals("1.2.3", RestResponseRedactor.redact("1.2.3")); - assertEquals("44", RestResponseRedactor.redact("44")); - } - - @Test - void masksEc2HostName() { - assertEquals("", RestResponseRedactor.redact("ip-10-0-0-1")); - assertEquals("", RestResponseRedactor.redact("ip-172-31-255-9")); - assertEquals("node here", RestResponseRedactor.redact("node ip-10-1-2-3 here")); - assertEquals("ip-256-0-0-1", RestResponseRedactor.redact("ip-256-0-0-1")); - } - - @Test - void masksFullIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80:0:0:0:0:0:0:1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("FE80:0:0:0:0:0:0:1")); - } - - @Test - void masksCompressedIpv6() { - assertEquals("x.x.x.x", RestResponseRedactor.redact("::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("fe80::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::1")); - assertEquals("x.x.x.x", RestResponseRedactor.redact("2001:db8::8a2e:370:7334")); - } - - @Test - void masksInetAddress() { - assertEquals("inet[/x.x.x.x:9200]", RestResponseRedactor.redact("inet[/10.0.0.7:9200]")); - } - - @Test - void masksAvailabilityZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-east-1a")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("ap-southeast-2b")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("eu-west-1c")); - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("us-gov-west-1a")); - // Shape-based match covers regions not in any hard-coded list (e.g. mx-central-1). - assertEquals("xx-xxxxx-xx", RestResponseRedactor.redact("mx-central-1a")); - assertEquals( - "a xx-xxxxx-xx b xx-xxxxx-xx", RestResponseRedactor.redact("a us-east-1a b us-west-2b")); - } - - @Test - void maskAvailabilityZoneMasksOnlyZones() { - assertEquals("xx-xxxxx-xx", RestResponseRedactor.maskAvailabilityZone("us-east-1a")); - assertEquals("10.0.0.7", RestResponseRedactor.maskAvailabilityZone("10.0.0.7")); - assertEquals("ip-10-0-0-1", RestResponseRedactor.maskAvailabilityZone("ip-10-0-0-1")); - } - - @Test - void leavesNonAddressesIntact() { - assertEquals( - "e4e136ea81e27370ff73cf753ba22d39", - RestResponseRedactor.redact("e4e136ea81e27370ff73cf753ba22d39")); - assertEquals( - "data,ingest,remote_cluster_client", - RestResponseRedactor.redact("data,ingest,remote_cluster_client")); - assertEquals( - "x.x.x.x 44 95 imr - e4e136ea", - RestResponseRedactor.redact("10.0.0.7 44 95 imr - e4e136ea")); - } - - @Test - void handlesNullAndEmpty() { - assertEquals(null, RestResponseRedactor.redact(null)); - assertEquals("", RestResponseRedactor.redact("")); - assertEquals(null, RestResponseRedactor.maskAvailabilityZone(null)); - assertEquals("", RestResponseRedactor.maskAvailabilityZone("")); - } -} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java similarity index 76% rename from opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java rename to opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java index df81225afbe..0b0aa1ec521 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchCatalogTableTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/system/OpenSearchSystemIndexTest.java @@ -32,12 +32,8 @@ import org.opensearch.sql.planner.physical.ProjectOperator; import org.opensearch.sql.storage.Table; -/** - * Covers the generic {@link OpenSearchCatalogTable} through a {@link SystemIndexCatalogSource}: - * schema delegation, the predefined-table contract, and the V2 physical path. - */ @ExtendWith(MockitoExtension.class) -class OpenSearchCatalogTableTest { +class OpenSearchSystemIndexTest { @Mock private OpenSearchClient client; @@ -45,37 +41,36 @@ class OpenSearchCatalogTableTest { @Mock private Settings settings; - private OpenSearchCatalogTable systemTable(String name) { - return new OpenSearchCatalogTable(new SystemIndexCatalogSource(client, name), settings); - } - @Test void testGetFieldTypesOfMetaTable() { - final Map fieldTypes = systemTable(TABLE_INFO).getFieldTypes(); + OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + final Map fieldTypes = systemIndex.getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("TABLE_CAT", STRING))); } @Test void testGetFieldTypesOfMappingTable() { - final Map fieldTypes = - systemTable(mappingTable("test_index")).getFieldTypes(); + OpenSearchSystemIndex systemIndex = + new OpenSearchSystemIndex(client, settings, mappingTable("test_index")); + final Map fieldTypes = systemIndex.getFieldTypes(); assertThat(fieldTypes, anyOf(hasEntry("COLUMN_NAME", STRING))); } @Test void testIsExist() { - assertTrue(systemTable(TABLE_INFO).exists()); + Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); + assertTrue(systemIndex.exists()); } @Test void testCreateTable() { - Table systemIndex = systemTable(TABLE_INFO); + Table systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); assertThrows(UnsupportedOperationException.class, () -> systemIndex.create(ImmutableMap.of())); } @Test void implement() { - OpenSearchCatalogTable systemIndex = systemTable(TABLE_INFO); + OpenSearchSystemIndex systemIndex = new OpenSearchSystemIndex(client, settings, TABLE_INFO); NamedExpression projectExpr = named("TABLE_NAME", ref("TABLE_NAME", STRING)); final PhysicalPlan plan = diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java index d9437fded5b..e1278aa75e8 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/SQLPlugin.java @@ -198,10 +198,6 @@ public List getRestHandlers( Metrics.getInstance().registerDefaultMetrics(); - // Publish the node SettingsFilter so the in-cluster `rest '/_cluster/settings'` fetcher can - // redact filtered settings exactly as the native GET /_cluster/settings endpoint does. - org.opensearch.sql.opensearch.storage.rest.RestSettingsFilterHolder.set(settingsFilter); - return Arrays.asList( new RestPPLQueryAction(), new RestPPLGrammarAction(), diff --git a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java index 5685d539541..f8214096e41 100644 --- a/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java +++ b/plugin/src/main/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryAction.java @@ -107,14 +107,13 @@ public boolean isAnalyticsIndex(String query, QueryType queryType) { .equals( IndicesService.CLUSTER_PLUGGABLE_DATAFORMAT_VALUE_SETTING.get( clusterService.getSettings()))) { - // Analytics engine serves neither the system catalog nor the rest command's reserved - // in-cluster source; both fall back to the default (Calcite) pipeline. + // Analytics engine can't serve system catalog; SHOW/DESCRIBE fall back to default pipeline try (UnifiedQueryContext context = buildParsingContext(queryType)) { - boolean defaultPipeline = + boolean systemCatalog = extractIndexName(query, queryType, context) - .map(name -> isSystemCatalog(name) || SystemIndexUtils.isRestSource(name)) + .map(RestUnifiedQueryAction::isSystemCatalog) .orElse(false); - return !defaultPipeline; + return !systemCatalog; } catch (Exception e) { // Check legacy-syntax SHOW/DESCRIBE; otherwise let AE handle and surface the error. return !isLegacySystemCatalogQuery(query); diff --git a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java index 0cf87f0604e..111597bb587 100644 --- a/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java +++ b/plugin/src/test/java/org/opensearch/sql/plugin/rest/RestUnifiedQueryActionTest.java @@ -178,12 +178,6 @@ public void describeStatementNotRoutedToAnalyticsEngineUnderClusterComposite() { assertFalse(action.isAnalyticsIndex("DESCRIBE TABLES LIKE 'parquet_logs'", QueryType.SQL)); } - @Test - public void restCommandNotRoutedToAnalyticsEngineUnderClusterComposite() { - enableClusterComposite(); - assertFalse(action.isAnalyticsIndex("| rest '/_cluster/health'", QueryType.PPL)); - } - @Test public void dataQueryStillRoutesToAnalyticsUnderClusterComposite() { enableClusterComposite(); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index fab30b66c3f..91d06b1fcd1 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -13,8 +13,6 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; -REST: 'REST'; -TIMEOUT: 'TIMEOUT'; EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index fda8d66d135..e05892c5a9d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -45,7 +45,6 @@ subSearch // commands pplCommands : describeCommand - | restCommand | showDataSourcesCommand | searchCommand | multisearchCommand @@ -106,7 +105,6 @@ commands commandName : SEARCH | DESCRIBE - | REST | SHOW | WHERE | FIELDS @@ -212,16 +210,6 @@ describeCommand : DESCRIBE tableSourceClause ; - -restCommand - : REST stringLiteral (restArgument)* - ; - -restArgument - : COUNT EQUAL integerLiteral - | TIMEOUT EQUAL stringLiteral - | ident EQUAL literalValue - ; showDataSourcesCommand : SHOW DATASOURCES ; @@ -1853,6 +1841,4 @@ searchableKeyWord | MAX_DEPTH | DEPTH_FIELD | EDGE - // rest command token, also usable as a free-text search term / identifier - | TIMEOUT ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e7d42793fe8..6d5ca92d167 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -113,7 +113,6 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; import org.opensearch.sql.ast.tree.ReplacePair; -import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -147,7 +146,6 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; import org.opensearch.sql.ppl.utils.UnresolvedPlanHelper; -import org.opensearch.sql.utils.SystemIndexUtils; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -259,37 +257,6 @@ public UnresolvedPlan visitShowDataSourcesCommand( return new DescribeRelation(qualifiedName(DATASOURCES_TABLE_NAME)); } - /** - * Rest command.
- * Leading command that reads an allow-listed, read-only in-cluster management endpoint - * (cluster/cat/nodes) as rows. The validated endpoint spec is encoded into a single reserved - * table name via {@link org.opensearch.sql.utils.SystemIndexUtils#restTable}; that name resolves - * through the storage engine to a REST source table on the Calcite path, mirroring how DESCRIBE - * resolves to a system index. Allow-list/authorization enforcement happens at source-table - * construction in the storage engine (it owns the transport actions and per-endpoint schemas). - */ - @Override - public UnresolvedPlan visitRestCommand(OpenSearchPPLParser.RestCommandContext ctx) { - String endpoint = StringUtils.unquoteText(ctx.stringLiteral().getText()); - LinkedHashMap args = new LinkedHashMap<>(); - Integer count = null; - String timeout = null; - for (OpenSearchPPLParser.RestArgumentContext arg : ctx.restArgument()) { - if (arg.COUNT() != null) { - count = Integer.parseInt(arg.integerLiteral().getText()); - } else if (arg.TIMEOUT() != null) { - timeout = StringUtils.unquoteText(arg.stringLiteral().getText()); - } else { - args.put( - StringUtils.unquoteIdentifier(arg.ident().getText()), - StringUtils.unquoteText(arg.literalValue().getText())); - } - } - String token = - SystemIndexUtils.restTable(new SystemIndexUtils.RestSpec(endpoint, args, count, timeout)); - return new RestRelation(new QualifiedName(token)); - } - /** Where command. */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 540869d1642..8adb5b79ce3 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -97,7 +97,6 @@ import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Replace; -import org.opensearch.sql.ast.tree.RestRelation; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; import org.opensearch.sql.ast.tree.SPath; @@ -125,7 +124,6 @@ import org.opensearch.sql.planner.logical.LogicalRemove; import org.opensearch.sql.planner.logical.LogicalRename; import org.opensearch.sql.planner.logical.LogicalSort; -import org.opensearch.sql.utils.SystemIndexUtils; /** Utility class to mask sensitive information in incoming PPL queries. */ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor { @@ -169,23 +167,6 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { - if (node instanceof RestRelation) { - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(node.getTableQualifiedName().toString()); - StringBuilder sb = new StringBuilder("rest ").append(spec.getEndpoint()); - if (spec.getCount() != null) { - sb.append(" count=").append(MASK_LITERAL); - } - if (spec.getTimeout() != null) { - sb.append(" timeout=").append(MASK_LITERAL); - } - if (spec.getArgs() != null) { - for (String key : spec.getArgs().keySet()) { - sb.append(' ').append(key).append('=').append(MASK_LITERAL); - } - } - return sb.toString(); - } if (node instanceof DescribeRelation) { return StringUtils.format("describe %s", MASK_TABLE); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java deleted file mode 100644 index da9424c9c8f..00000000000 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLRestTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ppl.calcite; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -import org.junit.Test; -import org.opensearch.sql.ast.Node; -import org.opensearch.sql.ast.tree.Project; -import org.opensearch.sql.ast.tree.RestRelation; -import org.opensearch.sql.common.setting.Settings; -import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; -import org.opensearch.sql.ppl.parser.AstBuilder; -import org.opensearch.sql.utils.SystemIndexUtils; - -/** - * Calcite-path coverage for the {@code rest} leading command at the parse / AST tier. - * - *

The {@code rest} row source resolves through {@code OpenSearchStorageEngine.getTable} -> - * {@code RestCatalogSource} -> {@code CalciteLogicalCatalogScan}, which lives in the {@code - * opensearch} module. This ppl-module Calcite harness binds a Calcite SCOTT schema rather than the - * OpenSearch storage engine, so the optimized {@code CalciteScannableCatalogScan} logical-plan - * assertion is exercised in {@code RestCatalogSourceTest} (source schema + request, unit) and - * {@code CalcitePPLRestIT} (schema + datarows on a live single-node cluster). This test pins the - * Calcite-facing contract that the ppl module owns: the grammar/AST rewrite of {@code rest} into a - * {@code RestRelation} carrying the validated, reserved-name-encoded endpoint spec that rides - * {@code visitRelation} exactly like {@code DESCRIBE}. - */ -public class CalcitePPLRestTest { - - private final PPLSyntaxParser parser = new PPLSyntaxParser(); - private final Settings settings = mock(Settings.class); - - private Node parse(String ppl) { - return new AstBuilder(ppl, settings).visit(parser.parse(ppl)); - } - - @Test - public void restHealthProjectsDeclaredColumns() { - Project project = - (Project) parse("| rest \"/_cluster/health\" | fields status, number_of_nodes"); - RestRelation rest = (RestRelation) project.getChild().get(0); - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); - assertEquals("/_cluster/health", spec.getEndpoint()); - // downstream fields compose on top of the rest row source. - assertEquals(2, project.getProjectList().size()); - } - - @Test - public void restReservedNameRoundTrips() { - RestRelation rest = - (RestRelation) parse("| rest \"/_cat/indices\" count=10 timeout=\"5s\" health=\"green\""); - String reserved = rest.getTableQualifiedName().toString(); - assertTrue(SystemIndexUtils.isRestSource(reserved)); - SystemIndexUtils.RestSpec spec = SystemIndexUtils.decodeRestSpec(reserved); - assertEquals("/_cat/indices", spec.getEndpoint()); - assertEquals(Integer.valueOf(10), spec.getCount()); - assertEquals("5s", spec.getTimeout()); - assertEquals("green", spec.getArgs().get("health")); - } -} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d57ca8a69bb..9d70487c741 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1939,32 +1939,4 @@ public void testJoinNoPrefixComparisonStaysCondition() { public void testJoinPrefixWithoutCriteriaKeywordIsSyntaxError() { assertThrows(SyntaxCheckException.class, () -> plan("source=t1 | inner join a t2")); } - - // rest command tests - - @Test - public void testRestCommand() { - org.opensearch.sql.ast.tree.Project project = - (org.opensearch.sql.ast.tree.Project) - plan("| rest \"/_cluster/health\" | fields status, number_of_nodes"); - org.opensearch.sql.ast.tree.RestRelation rest = - (org.opensearch.sql.ast.tree.RestRelation) project.getChild().get(0); - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); - assertEquals("/_cluster/health", spec.getEndpoint()); - assertTrue(spec.getArgs().isEmpty()); - } - - @Test - public void testRestCommandWithArgs() { - org.opensearch.sql.ast.tree.RestRelation rest = - (org.opensearch.sql.ast.tree.RestRelation) - plan("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\""); - SystemIndexUtils.RestSpec spec = - SystemIndexUtils.decodeRestSpec(rest.getTableQualifiedName().toString()); - assertEquals("/_cluster/health", spec.getEndpoint()); - assertEquals(Integer.valueOf(5), spec.getCount()); - assertEquals("30s", spec.getTimeout()); - assertEquals("indices", spec.getArgs().get("level")); - } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 1822b690feb..286f601b52c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -48,18 +48,6 @@ public void testPrometheusPPLCommand() { assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } - @Test - public void testRestCommand() { - assertEquals("rest /_cluster/health", anonymize("| rest \"/_cluster/health\"")); - } - - @Test - public void testRestCommandMasksArgValues() { - assertEquals( - "rest /_cluster/health count=*** timeout=*** level=***", - anonymize("| rest \"/_cluster/health\" count=5 timeout=\"30s\" level=\"indices\"")); - } - @Test public void testWhereCommand() { assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1"));