From 291de3264a8361aab2e90a0bd1fe333febab36a3 Mon Sep 17 00:00:00 2001 From: Pierre Tessier Date: Thu, 12 Dec 2019 09:37:26 -0800 Subject: [PATCH 1/5] clean up dd2 --- Convert/datadog.properties | 4 ++-- Convert/src/com/wavefront/labs/convert/Convert.java | 11 ++++++----- .../labs/convert/DefaultExpressionBuilder.java | 2 +- .../converter/datadog/DatadogTimeboardConverter.java | 1 - .../datadog/DatadogTimeboardConverter2.java | 1 - .../convert/converter/grafana/GrafanaConverter.java | 5 +++-- .../converter/grafana/GrafanaConverterHelper.java | 12 +++++++++++- .../converter/grafana/model/GrafanaPanelTarget.java | 9 +++++++++ .../labs/convert/converter/rrd/models/Def.java | 6 ++++-- .../labs/convert/converter/rrd/models/VDef.java | 3 ++- 10 files changed, 38 insertions(+), 16 deletions(-) diff --git a/Convert/datadog.properties b/Convert/datadog.properties index 027822f..6cb72d1 100644 --- a/Convert/datadog.properties +++ b/Convert/datadog.properties @@ -4,9 +4,9 @@ datadog.underscoreReplace=. datadog.dropTags=role,stackname,stack -convert.converter=com.wavefront.labs.convert.converter.datadog.DatadogConverter -convert.writer=com.wavefront.labs.convert.writer.FolderWriter +convert.converter=com.wavefront.labs.convert.converter.datadog.DatadogConverter2 convert.expressionBuilder=com.wavefront.labs.convert.converter.datadog.DatadogExpressionBuilder +convert.writer=com.wavefront.labs.convert.writer.FolderWriter convert.name.processor.file=datadog-name-processor.yaml convert.writer.tags=wf-converted diff --git a/Convert/src/com/wavefront/labs/convert/Convert.java b/Convert/src/com/wavefront/labs/convert/Convert.java index 34fa5af..e233772 100644 --- a/Convert/src/com/wavefront/labs/convert/Convert.java +++ b/Convert/src/com/wavefront/labs/convert/Convert.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -38,7 +39,7 @@ public void start(String[] args) { doWrite(models); - } catch (IOException | InstantiationException | ClassNotFoundException | IllegalAccessException e) { + } catch (IOException | InstantiationException | ClassNotFoundException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { logger.error("Fatal error in start.", e); } @@ -46,10 +47,10 @@ public void start(String[] args) { logger.error(com.wavefront.labs.convert.utils.Tracker.map); } - private List doConvert(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException { + private List doConvert(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException, NoSuchMethodException, InvocationTargetException { logger.info("Start Conversion"); - Converter converter = (Converter) Class.forName(properties.getProperty("convert.converter")).newInstance(); + Converter converter = (Converter) Class.forName(properties.getProperty("convert.converter")).getDeclaredConstructor().newInstance(); converter.init(properties); String filename = null; @@ -83,11 +84,11 @@ private List doConvert(String[] args) throws ClassNotFoundException, IllegalAcce return converter.convert(); } - private void doWrite(List models) throws ClassNotFoundException, IllegalAccessException, InstantiationException { + private void doWrite(List models) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { logger.info("Start Writing"); String generatorName = properties.getProperty("convert.writer", "com.wavefront.labs.convert.writer.WavefrontPublisher"); - Writer writer = (Writer) Class.forName(generatorName).newInstance(); + Writer writer = (Writer) Class.forName(generatorName).getDeclaredConstructor().newInstance(); writer.init(properties); // tags can be separated by whitespace, comma, or semi-colon diff --git a/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java b/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java index 853b494..ee04fb6 100644 --- a/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java +++ b/Convert/src/com/wavefront/labs/convert/DefaultExpressionBuilder.java @@ -20,7 +20,7 @@ public class DefaultExpressionBuilder implements ExpressionBuilder { protected Properties properties; - private NameProcessor nameProcessor; + protected NameProcessor nameProcessor; @Override public void init(Properties properties) { diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java index 90bb741..9a86703 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter.java @@ -208,7 +208,6 @@ private DashboardParameterValue createDashboardParameter(Variable variable) { //dashboardParameterValue.setDynamicFieldType(DashboardParameterValue.DynamicFieldTypeEnum.TAG_KEY); //dashboardParameterValue.setTagKey(variable.getValue()); //dashboardParameterValue.setQueryValue("ts(query.filter)"); - dashboardParameterValue.setValue(""); return dashboardParameterValue; } diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java index c2dd8c3..22def53 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogTimeboardConverter2.java @@ -211,7 +211,6 @@ private DashboardParameterValue createDashboardParameter(Variable variable) { //dashboardParameterValue.setDynamicFieldType(DashboardParameterValue.DynamicFieldTypeEnum.TAG_KEY); //dashboardParameterValue.setTagKey(variable.getValue()); //dashboardParameterValue.setQueryValue("ts(query.filter)"); - dashboardParameterValue.setValue(""); return dashboardParameterValue; } diff --git a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java index b95c6df..e3da80c 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java +++ b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverter.java @@ -17,6 +17,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -38,8 +39,8 @@ public void init(Properties properties) { String expressionBuilderClass = properties.getProperty("convert.expressionBuilder", ""); if (!expressionBuilderClass.equals("")) { try { - expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).newInstance(); - } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { + expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) { logger.error("Could not create instance of: " + expressionBuilderClass, e); expressionBuilder = new SimpleExpressionBuilder(); } diff --git a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java index d0d6131..032b462 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java +++ b/Convert/src/com/wavefront/labs/convert/converter/grafana/GrafanaConverterHelper.java @@ -262,7 +262,17 @@ private List fromTargets(List targets) { chartSourceQuery.setDisabled(target.isHide()); chartSourceQuery.setName(target.getRefId()); - String query = target.getTargetFull() != null && !target.getTargetFull().equals("") ? target.getTargetFull() : target.getTarget(); + String query; + if (target.getExpr() != null && !target.getExpr().equals("")) { + query = target.getExpr(); + } else if (target.getTargetFull() != null && !target.getTargetFull().equals("")) { + query = target.getTargetFull(); + } else if (target.getTarget() != null && !target.getTarget().equals("")){ + query = target.getTarget(); + } else { + continue; + } + chartSourceQuery.setQuery(expressionBuilder.buildExpression(query)); chartSourceQueries.add(chartSourceQuery); diff --git a/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java b/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java index 4befcf3..ff1f9f7 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java +++ b/Convert/src/com/wavefront/labs/convert/converter/grafana/model/GrafanaPanelTarget.java @@ -10,6 +10,7 @@ public class GrafanaPanelTarget { private String refId; private String target; private String targetFull; + private String expr; private boolean textEditor; public boolean isHide() { @@ -52,6 +53,14 @@ public void setTargetFull(String targetFull) { this.targetFull = targetFull; } + public String getExpr() { + return expr; + } + + public void setExpr(String expr) { + this.expr = expr; + } + public boolean isTextEditor() { return textEditor; } diff --git a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java index 0f77315..bc96476 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java +++ b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/Def.java @@ -6,6 +6,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.lang.reflect.InvocationTargetException; + public class Def extends Definition { private static final Logger logger = LogManager.getLogger(Def.class); @@ -35,8 +37,8 @@ public String calculate(RRDContext context) { String expressionBuilderClass = context.getProperties().getProperty("convert.expressionBuilder", ""); if (!expressionBuilderClass.equals("")) { try { - expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).newInstance(); - } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { + expressionBuilder = (ExpressionBuilder) Class.forName(expressionBuilderClass).getDeclaredConstructor().newInstance(); + } catch (IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | InvocationTargetException e) { logger.error("Could not create instance of: " + expressionBuilderClass, e); expressionBuilder = new RRDExpressionBuilder(); } diff --git a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java index 1d178cb..acc9910 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java +++ b/Convert/src/com/wavefront/labs/convert/converter/rrd/models/VDef.java @@ -12,6 +12,7 @@ public class VDef extends Definition { private static HashMap, String>> functionMap = new HashMap(); + static { functionMap.put("MAXIMUM", NotSupported::warning); functionMap.put("MINIMUM", NotSupported::warning); @@ -27,7 +28,7 @@ public class VDef extends Definition { functionMap.put("LSLCORREL", NotSupported::warning); } - public VDef (String line) { + public VDef(String line) { super(line); } From fb854e6237a3489384a58a412c45dc93ca22d54e Mon Sep 17 00:00:00 2001 From: Jenny Kim Date: Tue, 17 Dec 2019 16:03:54 -0500 Subject: [PATCH 2/5] Add a build script for swagger-codegen - Add missing dependencies to build.gradle [#170059714] Signed-off-by: Jesse Malone Co-authored-by: Jesse Malone --- Convert/build.gradle | 8 +++++++- Convert/build.sh | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100755 Convert/build.sh diff --git a/Convert/build.gradle b/Convert/build.gradle index 35b4636..b88b1a1 100644 --- a/Convert/build.gradle +++ b/Convert/build.gradle @@ -33,7 +33,13 @@ dependencies compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.2' compile 'com.squareup.okhttp:okhttp:2.7.5' compile 'com.google.code.gson:gson:2.6.2' - compile project(':wavefront-java-sdk') + compile 'org.threeten:threetenbp:1.4.0' + compile group: 'io.swagger.core.v3', name: 'swagger-annotations', version: '2.1.0' + compile group: 'io.swagger.codegen.v3', name: 'swagger-codegen-generators', version: '1.0.14' + compile group: 'javax.annotation', name: 'javax.annotation-api', version: '1.2' + compile group: 'com.squareup.okhttp', name: 'logging-interceptor', version: '2.7.5' + compile group: 'io.gsonfire', name: 'gson-fire', version: '1.8.3' + compile group: 'com.wavefront', name: 'wavefront-sdk-java', version: '1.15' compile 'junit:junit:4.12' } diff --git a/Convert/build.sh b/Convert/build.sh new file mode 100755 index 0000000..5eeaacc --- /dev/null +++ b/Convert/build.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +pushd swagger +./generate.sh +popd + +mv swagger/src/main src/main +mv swagger/src/test src/test From 03f8bd0c32b39b3811b5eb517fe8e06370fcfaa7 Mon Sep 17 00:00:00 2001 From: Jenny Kim Date: Thu, 19 Dec 2019 10:51:25 -0500 Subject: [PATCH 3/5] Fix formatting of datadog metrics that use the "origin" tag. - Also fix importing rep Capacity*Memory metrics Signed-off-by: Jesse Malone --- Convert/datadog-name-processor.yaml | 14 +++++++++++++ .../datadog/DatadogExpressionBuilder.java | 21 +++++++++++++++---- .../models/DatadogGraphDefinition2.java | 2 +- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/Convert/datadog-name-processor.yaml b/Convert/datadog-name-processor.yaml index dffdc2b..dbec3be 100644 --- a/Convert/datadog-name-processor.yaml +++ b/Convert/datadog-name-processor.yaml @@ -25,3 +25,17 @@ rules: search : 'mysql.galera.wsrep_cluster_size' replace : 'mysql.galera.wsrep_cluster_size' + - rule : datadog.nozzle goes away + match : '^datadog\.nozzle\..*$' + search : '^datadog\.nozzle\.(.*)$' + replace : 'pws-prod-test.$1' + + - rule : CapacityRemaining gets MiB + match : '^.*\.Capacity[a-zA-Z]*Memory$' + search : '^(.*\.Capacity[a-zA-Z]*Memory)$' + replace : '$1.MiB' + +# - rule : origin goes away +# match : '^.*origin.*$' +# search : '^.*origin.*$' +# replace : 'found.origin' diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java index cc5a017..b91ce65 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java @@ -91,10 +91,23 @@ public void init(Properties properties) { variablesMap = new HashMap(); } - @Override - public String buildMetricName(String orig) { + + public String buildMetricName(DatadogQuery datadogQuery) { + String orig = datadogQuery.getMetric(); + List scopes = datadogQuery.getScopes(); + if (scopes != null && scopes.size() > 0) { + for (String scope : scopes) { + String[] scopeParts = scope.split(":"); + if (scopeParts.length > 1 && scopeParts[0].equals("origin")) { + String origin = scopeParts[1]; + String matcher = "^datadog\\.nozzle\\.(.*)$"; + String replace = "datadog.nozzle." + origin + ".$1"; + orig = orig.replaceAll(matcher,replace); + } + } + } String metricName = buildName(orig, "metric"); - metricName = metricName.replaceAll("_", underscoreReplace); + //metricName = metricName.replaceAll("_", underscoreReplace); return super.buildMetricName(metricName); } @@ -188,7 +201,7 @@ private String makeMetricQuery(DatadogQuery datadogQuery) { String query = datadogQuery.getNumeral(); if (datadogQuery.getMetric() != null && !"".equals(datadogQuery.getMetric())) { - query = "ts(\"" + buildMetricName(datadogQuery.getMetric()) + "\""; + query = "ts(\"" + buildMetricName(datadogQuery) + "\""; List scopes = datadogQuery.getScopes(); if (scopes != null && scopes.size() > 0) { diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java index 4165af0..e05aaab 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/models/DatadogGraphDefinition2.java @@ -4,7 +4,7 @@ import java.util.List; -@JsonIgnoreProperties({"autoscale", "status", "markers", "style", "group", "no_metric_hosts", "scope", "no_group_hosts", "notes", "show_legend", "legend_size", "text_align", "custom_unit", "size_by", "tick_pos", "color_by_groups", "node_type", "show_tick", "compare_to", "xaxis", "color_by", "events", "tick_edge", "q", "group_by", "content", "font_size", "background_color", "precision", "layout_type", "title_size", "title_align", "color", "viz_type", "url", "time", "text", "alert_id", "tags_execution", "sizing", "query", "sort", "tags", "event_size", "margin", "count", "check", "hide_zero_counts", "grouping", "unit", "start", "color_preference", "display_format"}) +@JsonIgnoreProperties({"show_last_triggered","autoscale", "status", "markers", "style", "group", "no_metric_hosts", "scope", "no_group_hosts", "notes", "show_legend", "legend_size", "text_align", "custom_unit", "size_by", "tick_pos", "color_by_groups", "node_type", "show_tick", "compare_to", "xaxis", "color_by", "events", "tick_edge", "q", "group_by", "content", "font_size", "background_color", "precision", "layout_type", "title_size", "title_align", "color", "viz_type", "url", "time", "text", "alert_id", "tags_execution", "sizing", "query", "sort", "tags", "event_size", "margin", "count", "check", "hide_zero_counts", "grouping", "unit", "start", "color_preference", "display_format"}) public class DatadogGraphDefinition2 { private String type; From 7cc41dbff08c96cf8ca831c33ad0463de55009fe Mon Sep 17 00:00:00 2001 From: Jesse Malone Date: Thu, 19 Dec 2019 17:27:17 -0500 Subject: [PATCH 4/5] Handle expressions grouped by parentheses - Strip left/right parens off the start or end of and expression and add it back when re-assembling the expressions into a query. --- .../datadog/DatadogExpressionBuilder.java | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java index b91ce65..fa99322 100644 --- a/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java +++ b/Convert/src/com/wavefront/labs/convert/converter/datadog/DatadogExpressionBuilder.java @@ -24,7 +24,9 @@ public class DatadogExpressionBuilder extends DefaultExpressionBuilder { private static final Pattern expressionListPattern = Pattern.compile("\\{.*?\\}|(\\+|-|\\*|\\/|;)"); private static final Pattern topConveniencePattern = Pattern.compile("^(top|bottom)(5|10|15|20)_?(mean|min|max|last|area|l2norm|norm)?$"); - private static final Pattern operatorNumberPattern = Pattern.compile("(\\+|-|\\*|/)|(-?\\d+([.]\\d+)?)"); + +// NOTE: Including '(',')' as operators so they get added back to the query after parsing + private static final Pattern operatorNumberPattern = Pattern.compile("(\\(|\\)|\\+|-|\\*|/)|(-?\\d+([.]\\d+)?)"); private static final HashMap> functionMap = new HashMap(); @@ -274,12 +276,31 @@ private ArrayList createQueryList(String expression) { ArrayList expressionList = new ArrayList(); Matcher matcher = expressionListPattern.matcher(expression); + Pattern lParenMatcher = Pattern.compile("^\\("); + Pattern rParenMatcher = Pattern.compile("\\)$"); int lastPos = 0; while (matcher.find()) { if (matcher.group(1) != null) { - expressionList.add(expression.substring(lastPos, matcher.start()).trim()); + String subexpression = expression.substring(lastPos, matcher.start()).trim(); + + if (lParenMatcher.matcher(subexpression).find()) { +// Strip left parenthesis off expression, to be added back later (similar to how operators are handled) + expressionList.add("("); + expressionList.add(subexpression.substring(1,subexpression.length())); + } + else if (rParenMatcher.matcher(subexpression).find()) { +// Strip right parenthesis off expression, to be added back later (similar to how operators are handled) + expressionList.add(subexpression.substring(0, subexpression.length() - 1)); + expressionList.add(")"); + + } + else { + expressionList.add(subexpression); + } + expressionList.add(matcher.group(1)); + lastPos = matcher.end(); } } From 9471ca8b97da69e9debbfe92582bca4c41b21574 Mon Sep 17 00:00:00 2001 From: Jesse Malone Date: Thu, 19 Dec 2019 17:29:16 -0500 Subject: [PATCH 5/5] Add ".b" to system.healthy metrics --- Convert/datadog-name-processor.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Convert/datadog-name-processor.yaml b/Convert/datadog-name-processor.yaml index dbec3be..a535cb8 100644 --- a/Convert/datadog-name-processor.yaml +++ b/Convert/datadog-name-processor.yaml @@ -34,6 +34,10 @@ rules: match : '^.*\.Capacity[a-zA-Z]*Memory$' search : '^(.*\.Capacity[a-zA-Z]*Memory)$' replace : '$1.MiB' + - rule : system.healthy gets ".b" + match : '^.*\.system.healthy$' + search : '^(.*\.system.healthy)$' + replace : '$1.b' # - rule : origin goes away # match : '^.*origin.*$'