Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SDTEST-415] Add test_session metric #207

Merged
merged 3 commits into from
Jul 30, 2024
Merged

[SDTEST-415] Add test_session metric #207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d3a5071
constants for provider tags
anmarchenko Jul 30, 2024
0a12ac3
add helper method to emit test_session metric
anmarchenko Jul 30, 2024
052dac8
emit test_session metric when test session started
anmarchenko Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions lib/datadog/ci/ext/environment.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,23 @@ module Environment
TAG_NODE_NAME = "ci.node.name"
TAG_CI_ENV_VARS = "_dd.ci.env_vars"

module Provider
APPVEYOR = "appveyor"
AZURE = "azurepipelines"
AWS = "awscodepipeline"
BITBUCKET = "bitbucket"
BITRISE = "bitrise"
BUDDYCI = "buddy"
BUILDKITE = "buildkite"
CIRCLECI = "circleci"
CODEFRESH = "codefresh"
GITHUB = "github"
GITLAB = "gitlab"
JENKINS = "jenkins"
TEAMCITY = "teamcity"
TRAVISCI = "travisci"
end

POSSIBLE_BUNDLE_LOCATIONS = %w[vendor/bundle .bundle].freeze

module_function
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/appveyor.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def self.handles?(env)
end

def provider_name
"appveyor"
Provider::APPVEYOR
end

def pipeline_url
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/aws_code_pipeline.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def self.handles?(env)
end

def provider_name
"awscodepipeline"
Provider::AWS
end

def pipeline_id
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/azure.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def self.handles?(env)
end

def provider_name
"azurepipelines"
Provider::AZURE
end

def pipeline_url
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/bitbucket.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def self.handles?(env)

# overridden methods
def provider_name
"bitbucket"
Provider::BITBUCKET
end

def pipeline_id
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/bitrise.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def self.handles?(env)
end

def provider_name
"bitrise"
Provider::BITRISE
end

def pipeline_id
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/buddy.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def self.handles?(env)
end

def provider_name
"buddy"
Provider::BUDDYCI
end

def pipeline_id
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/buildkite.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def self.handles?(env)
end

def provider_name
"buildkite"
Provider::BUILDKITE
end

def job_url
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/circleci.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def self.handles?(env)
end

def provider_name
"circleci"
Provider::CIRCLECI
end

def job_url
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/codefresh.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ def self.handles?(env)
end

def provider_name
"codefresh"
Provider::CODEFRESH
end

def job_name
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/github_actions.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def self.handles?(env)
end

def provider_name
"github"
Provider::GITHUB
end

def job_name
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/gitlab.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def self.handles?(env)
end

def provider_name
"gitlab"
Provider::GITLAB
end

def job_name
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/jenkins.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def self.handles?(env)
end

def provider_name
"jenkins"
Provider::JENKINS
end

def pipeline_id
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/teamcity.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def self.handles?(env)
end

def provider_name
"teamcity"
Provider::TEAMCITY
end

def job_name
Expand Down
2 changes: 1 addition & 1 deletion lib/datadog/ci/ext/environment/providers/travis.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def self.handles?(env)
end

def provider_name
"travisci"
Provider::TRAVISCI
end

def job_url
Expand Down
18 changes: 18 additions & 0 deletions lib/datadog/ci/ext/telemetry.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,24 @@ module Command
GET_OBJECTS = "get_objects"
PACK_OBJECTS = "pack_objects"
end

module Provider
APPVEYOR = "appveyor"
AWS = "aws"
AZURE = "azp"
BITBUCKET = "bitbucket"
BITRISE = "bitrise"
BUDDYCI = "buddyci"
BUILDKITE = "buildkite"
CIRCLECI = "circleci"
CODEFRESH = "codefresh"
GITHUB = "githubactions"
GITLAB = "gitlab"
JENKINS = "jenkins"
TEAMCITY = "teamcity"
TRAVISCI = "travisci"
UNSUPPORTED = "unsupported"
end
end
end
end
Expand Down
1 change: 1 addition & 0 deletions lib/datadog/ci/test_visibility/component.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ def itr_enabled?
# DOMAIN EVENTS
def on_test_session_started(test_session)
Telemetry.event_created(test_session)
Telemetry.test_session_started(test_session)

# finds and instruments additional test libraries that we support (ex: selenium-webdriver)
Contrib.auto_instrument_on_session_start!
Expand Down
31 changes: 31 additions & 0 deletions lib/datadog/ci/test_visibility/telemetry.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,23 @@ module Telemetry
Ext::AppTypes::TYPE_TEST_SESSION => Ext::Telemetry::EventType::SESSION
}.freeze

PROVIDER_TAG_TO_TELEMETRY_PROVIDER_TAG = {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is incredibly ugly: for some reason we use different values for some providers when submitting to citestcycle and telemetry. For example Microsoft Azure is "azurepipelines" in citestcylce but "azp" in telemetry

Copy link

@romainkomorndatadog romainkomorndatadog Jul 30, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hot take: I think we shouldn't do this and I think we should fix the other tracers to use the same tags for telemetry and citestcycle.

Unless I'm mistaken, the backend doesn't have an enum validating these values, so worst case, we'd have some extra values for some time.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like your proposal, but I also would like to move fast and leave this epic behind. Given that the discussion about naming can go on for months from now, let's merge this one as is and then I'll fix it in one of the subsequent PRs

Ext::Environment::Provider::APPVEYOR => Ext::Telemetry::Provider::APPVEYOR,
Ext::Environment::Provider::AWS => Ext::Telemetry::Provider::AWS,
Ext::Environment::Provider::AZURE => Ext::Telemetry::Provider::AZURE,
Ext::Environment::Provider::BITBUCKET => Ext::Telemetry::Provider::BITBUCKET,
Ext::Environment::Provider::BITRISE => Ext::Telemetry::Provider::BITRISE,
Ext::Environment::Provider::BUDDYCI => Ext::Telemetry::Provider::BUDDYCI,
Ext::Environment::Provider::BUILDKITE => Ext::Telemetry::Provider::BUILDKITE,
Ext::Environment::Provider::CIRCLECI => Ext::Telemetry::Provider::CIRCLECI,
Ext::Environment::Provider::CODEFRESH => Ext::Telemetry::Provider::CODEFRESH,
Ext::Environment::Provider::GITHUB => Ext::Telemetry::Provider::GITHUB,
Ext::Environment::Provider::GITLAB => Ext::Telemetry::Provider::GITLAB,
Ext::Environment::Provider::JENKINS => Ext::Telemetry::Provider::JENKINS,
Ext::Environment::Provider::TEAMCITY => Ext::Telemetry::Provider::TEAMCITY,
Ext::Environment::Provider::TRAVISCI => Ext::Telemetry::Provider::TRAVISCI
}.freeze

def self.event_created(span)
Utils::Telemetry.inc(Ext::Telemetry::METRIC_EVENT_CREATED, 1, event_tags_from_span(span))
end
Expand All @@ -28,6 +45,20 @@ def self.event_finished(span)
Utils::Telemetry.inc(Ext::Telemetry::METRIC_EVENT_FINISHED, 1, tags)
end

def self.test_session_started(test_session)
Utils::Telemetry.inc(
Ext::Telemetry::METRIC_TEST_SESSION,
1,
{
Ext::Telemetry::TAG_AUTO_INJECTED => "false", # ruby doesn't support auto injection yet
Ext::Telemetry::TAG_PROVIDER => PROVIDER_TAG_TO_TELEMETRY_PROVIDER_TAG.fetch(
test_session.get_tag(Ext::Environment::TAG_PROVIDER_NAME),
Ext::Telemetry::Provider::UNSUPPORTED
)
}
)
end

def self.event_tags_from_span(span)
# base tags for span
# @type var tags: Hash[String, String]
Expand Down
17 changes: 17 additions & 0 deletions sig/datadog/ci/ext/environment.rbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,23 @@ module Datadog

PROVIDERS: ::Array[Array[String | Symbol]]

module Provider
APPVEYOR: "appveyor"
AWS: "awscodepipeline"
AZURE: "azurepipelines"
BITBUCKET: "bitbucket"
BITRISE: "bitrise"
BUDDYCI: "buddy"
BUILDKITE: "buildkite"
CIRCLECI: "circleci"
CODEFRESH: "codefresh"
GITHUB: "github"
GITLAB: "gitlab"
JENKINS: "jenkins"
TEAMCITY: "teamcity"
TRAVISCI: "travisci"
end

def self?.tags: (untyped env) -> Hash[String, String]

def self?.ensure_post_conditions: (Hash[String, String] tags) -> void
Expand Down
18 changes: 18 additions & 0 deletions sig/datadog/ci/ext/telemetry.rbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,24 @@ module Datadog

PACK_OBJECTS: "pack_objects"
end

module Provider
APPVEYOR: "appveyor"
AWS: "aws"
AZURE: "azp"
BITBUCKET: "bitbucket"
BITRISE: "bitrise"
BUDDYCI: "buddyci"
BUILDKITE: "buildkite"
CIRCLECI: "circleci"
CODEFRESH: "codefresh"
GITHUB: "githubactions"
GITLAB: "gitlab"
JENKINS: "jenkins"
TEAMCITY: "teamcity"
TRAVISCI: "travisci"
UNSUPPORTED: "unsupported"
end
end
end
end
Expand Down
4 changes: 4 additions & 0 deletions sig/datadog/ci/test_visibility/telemetry.rbs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@ module Datadog
module Telemetry
SPAN_TYPE_TO_TELEMETRY_EVENT_TYPE: ::Hash[String, String]

PROVIDER_TAG_TO_TELEMETRY_PROVIDER_TAG: ::Hash[String, String]

def self.event_created: (Datadog::CI::Span span) -> void

def self.event_finished: (Datadog::CI::Span span) -> void

def self.test_session_started: (Datadog::CI::TestSession span) -> void

def self.event_tags_from_span: (Datadog::CI::Span span) -> ::Hash[String, String]

def self.add_browser_tags!: (Datadog::CI::Span span, ::Hash[String, String] tags) -> void
Expand Down
1 change: 1 addition & 0 deletions spec/datadog/ci/test_visibility/component_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@
end
end
it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_EVENT_CREATED
it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_TEST_SESSION
end

describe "#start_test_module" do
Expand Down
35 changes: 35 additions & 0 deletions spec/datadog/ci/test_visibility/telemetry_spec.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,4 +206,39 @@
it { event_finished }
end
end

describe ".test_session_started" do
subject(:test_session_started) { described_class.test_session_started(test_session) }

let(:provider_tag) { "github" }
let(:expected_provider_telemetry_tag) { Datadog::CI::Ext::Telemetry::Provider::GITHUB }

let(:test_session) do
instance_double(
Datadog::CI::TestSession,
get_tag: provider_tag
)
end

before do
expect(Datadog::CI::Utils::Telemetry).to receive(:inc)
.with(
Datadog::CI::Ext::Telemetry::METRIC_TEST_SESSION,
1,
{
Datadog::CI::Ext::Telemetry::TAG_AUTO_INJECTED => "false",
Datadog::CI::Ext::Telemetry::TAG_PROVIDER => expected_provider_telemetry_tag
}
)
end

it { test_session_started }

context "when provider is not supported" do
let(:provider_tag) { nil }
let(:expected_provider_telemetry_tag) { Datadog::CI::Ext::Telemetry::Provider::UNSUPPORTED }

it { test_session_started }
end
end
end
Loading