Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update metric normalization #1729

Merged
merged 6 commits into from
Apr 10, 2024
Merged

Update metric normalization #1729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c080bae
Update metric normalization
cleptric Apr 3, 2024
41c677b
Escape control characters
cleptric Apr 4, 2024
b0adca5
Escape `\` before adding `\` to string
stayallive Apr 4, 2024
ad3b8aa
Refactor tag value escaping
cleptric Apr 9, 2024
cc98543
Cleanup
cleptric Apr 10, 2024
735aed5
Merge branch 'master' into metric-normalization
cleptric Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 80 additions & 39 deletions src/Serializer/EnvelopItems/MetricsItem.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

use Sentry\Event;
use Sentry\Metrics\MetricsUnit;
use Sentry\Metrics\Types\AbstractType;
use Sentry\Serializer\Traits\StacktraceFrameSeralizerTrait;
use Sentry\Util\JSON;

Expand All @@ -19,12 +20,17 @@ class MetricsItem implements EnvelopeItemInterface
/**
* @var string
*/
private const KEY_PATTERN = '/[^a-zA-Z0-9_\/.-]+/i';
private const KEY_PATTERN = '/[^\w\-.]+/i';

/**
* @var string
*/
private const VALUE_PATTERN = '/[^\w\d\s_:\/@\.{}\[\]$-]+/i';
private const UNIT_PATTERN = '/[^\w]+/i';

/**
* @var string
*/
private const TAG_KEY_PATTERN = '/[^\w\-.\/]+/i';

public static function toEnvelopeItem(Event $event): string
{
Expand All @@ -37,43 +43,7 @@ public static function toEnvelopeItem(Event $event): string
$metricMetaPayload = [];

foreach ($metrics as $metric) {
/**
* In case of us adding support for emitting metrics from other namespaces,
* we have to alter the RateLimiter::class to properly handle these
* namespaces.
*/

// key - my.metric
$line = preg_replace(self::KEY_PATTERN, '_', $metric->getKey());

if ($metric->getUnit() !== MetricsUnit::none()) {
// unit - @second
$line .= '@' . $metric->getunit();
}

foreach ($metric->serialize() as $value) {
// value - 2:3:4...
$line .= ':' . $value;
}

// type - |c|, |d|, ...
$line .= '|' . $metric->getType() . '|';

$tags = [];
foreach ($metric->getTags() as $key => $value) {
$tags[] = preg_replace(self::KEY_PATTERN, '_', $key) .
':' . preg_replace(self::VALUE_PATTERN, '', $value);
}

if (!empty($tags)) {
// tags - #key:value,key:value...
$line .= '#' . implode(',', $tags) . '|';
}

// timestamp - T123456789
$line .= 'T' . $metric->getTimestamp();

$statsdPayload[] = $line;
$statsdPayload[] = self::seralizeMetric($metric);

if ($metric->hasCodeLocation()) {
$metricMetaPayload[$metric->getMri()][] = array_merge(
Expand Down Expand Up @@ -116,4 +86,75 @@ public static function toEnvelopeItem(Event $event): string
$statsdPayload
);
}

public static function seralizeMetric(AbstractType $metric): string
{
/**
* In case of us adding support for emitting metrics from other namespaces,
* we have to alter the RateLimiter::class to properly handle these
* namespaces.
*/

// key - my.metric
$line = preg_replace(self::KEY_PATTERN, '_', $metric->getKey());

if ($metric->getUnit() !== MetricsUnit::none()) {
// unit - @second
$line .= '@' . preg_replace(self::UNIT_PATTERN, '', (string) $metric->getUnit());
}

foreach ($metric->serialize() as $value) {
// value - 2:3:4...
$line .= ':' . $value;
}

// type - |c|, |d|, ...
$line .= '|' . $metric->getType() . '|';

$tags = [];
foreach ($metric->getTags() as $key => $value) {
$tags[] = preg_replace(self::TAG_KEY_PATTERN, '', $key) .
':' . self::escapeTagValues($value);
}

if (!empty($tags)) {
// tags - #key:value,key:value...
$line .= '#' . implode(',', $tags) . '|';
}

// timestamp - T123456789
$line .= 'T' . $metric->getTimestamp();

return $line;
}

public static function escapeTagValues(string $tagValue): string
{
$result = '';

for ($i = 0; $i < mb_strlen($tagValue); ++$i) {
$character = mb_substr($tagValue, $i, 1);
$result .= str_replace(
[
"\n",
"\r",
"\t",
'\\',
'|',
',',
],
[
'\n',
'\r',
'\t',
'\\\\',
'\u{7c}',
'\u{2c}',
],
$character
);
}

return $result;
}
}
30 changes: 30 additions & 0 deletions tests/Serializer/EnvelopeItems/MetricsItemTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
<?php

declare(strict_types=1);

namespace Sentry\Tests\Serializer;

use PHPUnit\Framework\TestCase;
use Sentry\Serializer\EnvelopItems\MetricsItem;

final class MetricsItemTest extends TestCase
{
public function testEscapeTagValues(): void
{
// No escaping
$this->assertSame('plain', MetricsItem::escapeTagValues('plain'));
$this->assertSame('plain text', MetricsItem::escapeTagValues('plain text'));
$this->assertSame('plain%text', MetricsItem::escapeTagValues('plain%text'));

// Escape sequences
$this->assertSame('plain \\\\\\\\ text', MetricsItem::escapeTagValues('plain \\\\ text'));
$this->assertSame('plain\\u{2c}text', MetricsItem::escapeTagValues('plain,text'));
$this->assertSame('plain\\u{7c}text', MetricsItem::escapeTagValues('plain|text'));
$this->assertSame('plain 😅', MetricsItem::escapeTagValues('plain 😅'));

// Escapable control characters
$this->assertSame('plain\\\\ntext', MetricsItem::escapeTagValues("plain\ntext"));
$this->assertSame('plain\\\\rtext', MetricsItem::escapeTagValues("plain\rtext"));
$this->assertSame('plain\\\\ttext', MetricsItem::escapeTagValues("plain\ttext"));
}
}
14 changes: 7 additions & 7 deletions tests/Serializer/PayloadSerializerTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,9 +415,9 @@ public static function serializeAsEnvelopeDataProvider(): iterable
];

$counter = new CounterType('counter', 1.0, MetricsUnit::second(), ['foo' => 'bar', 'route' => 'GET /foo'], 1597790835);
$distribution = new DistributionType('distribution', 1.0, MetricsUnit::second(), ['$foo$' => '%bar%'], 1597790835);
$gauge = new GaugeType('gauge', 1.0, MetricsUnit::second(), ['föö' => 'bär'], 1597790835);
$set = new SetType('set', 1.0, MetricsUnit::second(), ['%{key}' => '$value$'], 1597790835);
$distribution = new DistributionType('distribution', 1.0, MetricsUnit::second(), ['foo' => 'bar'], 1597790835);
$gauge = new GaugeType('gauge', 1.0, MetricsUnit::second(), ['foo' => 'bar', 'bar' => 'baz'], 1597790835);
$set = new SetType('set', 1.0, MetricsUnit::second(), ['key' => 'value'], 1597790835);
$noTags = new CounterType('no_tags', 1.0, MetricsUnit::second(), [], 1597790835);

$event = Event::createMetrics(new EventId('fc9442f5aef34234bb22b9a615e30ccd'));
Expand All @@ -433,11 +433,11 @@ public static function serializeAsEnvelopeDataProvider(): iterable
$event,
<<<TEXT
{"event_id":"fc9442f5aef34234bb22b9a615e30ccd","sent_at":"2020-08-18T22:47:15Z","dsn":"http:\/\/public@example.com\/sentry\/1","sdk":{"name":"sentry.php","version":"$sdkVersion"}}
{"type":"statsd","length":218}
{"type":"statsd","length":222}
counter@second:1|c|#foo:bar,route:GET /foo|T1597790835
distribution@second:1|d|#_foo_:bar|T1597790835
gauge@second:1:1:1:1:1|g|#f_:br|T1597790835
set@second:1|s|#_key_:\$value\$|T1597790835
distribution@second:1|d|#foo:bar|T1597790835
gauge@second:1:1:1:1:1|g|#foo:bar,bar:baz|T1597790835
set@second:1|s|#key:value|T1597790835
no_tags@second:1|c|T1597790835
TEXT
,
Expand Down
Loading