Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@
package org.opensearch.sql.api;

import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_ENGINE_ENABLED;
import static org.opensearch.sql.common.setting.Settings.Key.PATTERN_BUFFER_LIMIT;
import static org.opensearch.sql.common.setting.Settings.Key.PATTERN_MAX_SAMPLE_COUNT;
import static org.opensearch.sql.common.setting.Settings.Key.PATTERN_METHOD;
import static org.opensearch.sql.common.setting.Settings.Key.PATTERN_MODE;
import static org.opensearch.sql.common.setting.Settings.Key.PATTERN_SHOW_NUMBERED_TOKEN;
import static org.opensearch.sql.common.setting.Settings.Key.PPL_JOIN_SUBSEARCH_MAXOUT;
import static org.opensearch.sql.common.setting.Settings.Key.PPL_REX_MAX_MATCH_LIMIT;
import static org.opensearch.sql.common.setting.Settings.Key.PPL_SUBSEARCH_MAXOUT;
Expand Down Expand Up @@ -145,12 +150,18 @@ public static class Builder {
*/
private final Map<Settings.Key, Object> settings =
new HashMap<Settings.Key, Object>(
Map.of(
QUERY_SIZE_LIMIT, SysLimit.DEFAULT.querySizeLimit(),
PPL_SUBSEARCH_MAXOUT, SysLimit.UNLIMITED_SUBSEARCH.subsearchLimit(),
PPL_JOIN_SUBSEARCH_MAXOUT, SysLimit.UNLIMITED_SUBSEARCH.joinSubsearchLimit(),
CALCITE_ENGINE_ENABLED, true,
PPL_REX_MAX_MATCH_LIMIT, 10));
Map.ofEntries(
Comment thread
dai-chen marked this conversation as resolved.
Map.entry(QUERY_SIZE_LIMIT, SysLimit.DEFAULT.querySizeLimit()),
Map.entry(PPL_SUBSEARCH_MAXOUT, SysLimit.UNLIMITED_SUBSEARCH.subsearchLimit()),
Map.entry(
PPL_JOIN_SUBSEARCH_MAXOUT, SysLimit.UNLIMITED_SUBSEARCH.joinSubsearchLimit()),
Map.entry(CALCITE_ENGINE_ENABLED, true),
Map.entry(PPL_REX_MAX_MATCH_LIMIT, 10),
Map.entry(PATTERN_METHOD, "SIMPLE_PATTERN"),
Map.entry(PATTERN_MODE, "LABEL"),
Map.entry(PATTERN_MAX_SAMPLE_COUNT, 10),
Map.entry(PATTERN_BUFFER_LIMIT, 100000),
Map.entry(PATTERN_SHOW_NUMBERED_TOKEN, false)));

/**
* Sets the query language frontend to be used.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,4 +144,18 @@ public void assertionErrorIsWrappedAsSemanticCheckException() {
.assertErrorMessageEquals("Failed to plan query: invalid plan structure")
.assertCauseType(AssertionError.class);
}

/**
* Without the {@code PATTERN_*} defaults in {@link UnifiedQueryContext}, a bare {@code patterns
* <field>} (no explicit {@code method=}/{@code mode=}) dies at parse time with {@code
* PatternMethod.valueOf("NULL")} because {@code AstBuilder.visitPatternsCommand} reads a null
* from {@code settings.getSettingValue(Key.PATTERN_METHOD)}. With the defaults present, the
* planner lowers patterns to SIMPLE / LABEL mode and adds {@code patterns_field}.
*/
@Test
public void testPPLPatternsPicksUpDefaults() {
givenQuery("source = catalog.employees | patterns name")
.assertPlanContains("REGEXP_REPLACE")
.assertFields("id", "name", "age", "department", "patterns_field");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
CalcitePPLCastFunctionIT.class,
CalcitePPLConditionBuiltinFunctionIT.class,
CalcitePPLCryptographicFunctionIT.class,
CalcitePPLDashboardPatternsIT.class,
CalcitePPLDedupIT.class,
CalcitePPLEventstatsIT.class,
CalciteStreamstatsCommandIT.class,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.calcite.remote;

import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_HDFS_LOGS;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.schema;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder;

import com.google.common.collect.ImmutableList;
import java.io.IOException;
import org.json.JSONObject;
import org.junit.Test;
import org.opensearch.sql.ppl.PPLIntegTestCase;

/** Pins the BRAIN-label pattern panel query shape used by OpenSearch Dashboards. */
public class CalcitePPLDashboardPatternsIT extends PPLIntegTestCase {
@Override
public void init() throws Exception {
super.init();
enableCalcite();
loadIndex(Index.HDFS_LOGS);
}

@Test
public void testDashboardBrainLabelStatsByPatternsField() throws IOException {
JSONObject result =
executeQuery(
String.format(
"source=%s"
+ " | patterns content method=BRAIN mode=label"
+ " max_sample_count=5 variable_count_threshold=5"
+ " frequency_threshold_percentage=0.2"
+ " | stats count() as pattern_count, take(content, 1) as sample_logs"
+ " by patterns_field"
+ " | sort - pattern_count"
+ " | fields patterns_field, pattern_count, sample_logs",
TEST_INDEX_HDFS_LOGS));
verifySchemaInOrder(
Comment thread
dai-chen marked this conversation as resolved.
result,
schema("patterns_field", "string"),
schema("pattern_count", "bigint"),
schema("sample_logs", "array"));
verifyDataRows(
result,
rows(
"BLOCK* NameSystem.addStoredBlock: blockMap updated: <*IP*> is added to blk_<*> size"
+ " <*>",
2,
ImmutableList.of(
"BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.31.85:50010 is added"
+ " to blk_-7017553867379051457 size 67108864")),
rows(
"PacketResponder failed <*> blk_<*>",
2,
ImmutableList.of("PacketResponder failed for blk_6996194389878584395")),
rows(
"Verification succeeded <*> blk_<*>",
2,
ImmutableList.of("Verification succeeded for blk_-1547954353065580372")),
rows(
"<*> NameSystem.allocateBlock:"
+ " /user/root/sortrand/_temporary/_task_<*>_<*>_r_<*>_<*>/part<*> blk_<*>",
2,
ImmutableList.of(
"BLOCK* NameSystem.allocateBlock:"
+ " /user/root/sortrand/_temporary/_task_200811092030_0002_r_000296_0/part-00296."
+ " blk_-6620182933895093708")));
}
}
Loading