Skip to content

Commit

Permalink
Extract local git extractor and move normalization into extractor
Browse files Browse the repository at this point in the history
  • Loading branch information
@anmarchenko
anmarchenko committed Sep 4, 2023
1 parent f3be086 commit fd3ed19
Show file tree
Hide file tree
Showing 12 changed files with 244 additions and 164 deletions.
165 changes: 4 additions & 161 deletions lib/datadog/ci/ext/environment.rb
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
# frozen_string_literal: true

require "open3"
require "json"

require_relative "git"
require_relative "environment/extractor"
require_relative "environment/user_defined_tags"
require_relative "environment/local_git"

module Datadog
module CI
Expand Down Expand Up @@ -36,168 +34,13 @@ def tags(env)
UserDefinedTags.new(env).tags
)

# Normalize Git references
if !tags[Git::TAG_BRANCH].nil? && tags[Git::TAG_BRANCH].include?("tags/")
tags[Git::TAG_TAG] = tags[Git::TAG_BRANCH]
tags.delete(Git::TAG_BRANCH)
end
tags[Git::TAG_TAG] = normalize_ref(tags[Git::TAG_TAG])
tags[Git::TAG_BRANCH] = normalize_ref(tags[Git::TAG_BRANCH])
tags[Git::TAG_REPOSITORY_URL] = filter_sensitive_info(
tags[Git::TAG_REPOSITORY_URL]
)

# Expand ~
workspace_path = tags[TAG_WORKSPACE_PATH]
if !workspace_path.nil? && (workspace_path == "~" || workspace_path.start_with?("~/"))
tags[TAG_WORKSPACE_PATH] = File.expand_path(workspace_path)
end

# Fill out tags from local git as fallback
extract_local_git.each do |key, value|
local_git_tags = LocalGit.new(env).tags
local_git_tags.each do |key, value|
tags[key] ||= value
end

tags.reject { |_, v| v.nil? }
end

def normalize_ref(name)
refs = %r{^refs/(heads/)?}
origin = %r{^origin/}
tags = %r{^tags/}
name.gsub(refs, "").gsub(origin, "").gsub(tags, "") unless name.nil?
end

def filter_sensitive_info(url)
url.gsub(%r{(https?://)[^/]*@}, '\1') unless url.nil?
end

# CI providers
def git_commit_users
# Get committer and author information in one command.
output = exec_git_command("git show -s --format='%an\t%ae\t%at\t%cn\t%ce\t%ct'")
return unless output

fields = output.split("\t").each(&:strip!)

{
author_name: fields[0],
author_email: fields[1],
# Because we can't get a reliable UTC time from all recent versions of git
# We have to rely on converting the date to UTC ourselves.
author_date: Time.at(fields[2].to_i).utc.to_datetime.iso8601,
committer_name: fields[3],
committer_email: fields[4],
# Because we can't get a reliable UTC time from all recent versions of git
# We have to rely on converting the date to UTC ourselves.
committer_date: Time.at(fields[5].to_i).utc.to_datetime.iso8601
}
rescue => e
Datadog.logger.debug(
"Unable to read git commit users: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
)
nil
end

def git_repository_url
exec_git_command("git ls-remote --get-url")
rescue => e
Datadog.logger.debug(
"Unable to read git repository url: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
)
nil
end

def git_commit_message
exec_git_command("git show -s --format=%s")
rescue => e
Datadog.logger.debug(
"Unable to read git commit message: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
)
nil
end

def git_branch
exec_git_command("git rev-parse --abbrev-ref HEAD")
rescue => e
Datadog.logger.debug(
"Unable to read git branch: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
)
nil
end

def git_commit_sha
exec_git_command("git rev-parse HEAD")
rescue => e
Datadog.logger.debug(
"Unable to read git commit SHA: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
)
nil
end

def git_tag
exec_git_command("git tag --points-at HEAD")
rescue => e
Datadog.logger.debug(
"Unable to read git tag: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
)
nil
end

def git_base_directory
exec_git_command("git rev-parse --show-toplevel")
rescue => e
Datadog.logger.debug(
"Unable to read git base directory: #{e.class.name} #{e.message} at #{Array(e.backtrace).first}"
)
nil
end

def exec_git_command(cmd)
out, status = Open3.capture2e(cmd)

raise "Failed to run git command #{cmd}: #{out}" unless status.success?

out.strip! # There's always a "\n" at the end of the command output

return nil if out.empty?

out
end

def extract_local_git
env = {
TAG_WORKSPACE_PATH => git_base_directory,
Git::TAG_REPOSITORY_URL => git_repository_url,
Git::TAG_COMMIT_SHA => git_commit_sha,
Git::TAG_BRANCH => git_branch,
Git::TAG_TAG => git_tag,
Git::TAG_COMMIT_MESSAGE => git_commit_message
}

if (commit_users = git_commit_users)
env.merge!(
Git::TAG_COMMIT_AUTHOR_NAME => commit_users[:author_name],
Git::TAG_COMMIT_AUTHOR_EMAIL => commit_users[:author_email],
Git::TAG_COMMIT_AUTHOR_DATE => commit_users[:author_date],
Git::TAG_COMMIT_COMMITTER_NAME => commit_users[:committer_name],
Git::TAG_COMMIT_COMMITTER_EMAIL => commit_users[:committer_email],
Git::TAG_COMMIT_COMMITTER_DATE => commit_users[:committer_date]
)
end

env
end

def branch_or_tag(branch_or_tag)
branch = tag = nil
if branch_or_tag && branch_or_tag.include?("tags/")
tag = branch_or_tag
else
branch = branch_or_tag
end

[branch, tag]
tags
end
end
end
Expand Down
41 changes: 38 additions & 3 deletions lib/datadog/ci/ext/environment/extractor.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def initialize(env)
end

def tags
{
tags = {
Environment::TAG_JOB_NAME => job_name,
Environment::TAG_JOB_URL => job_url,
Environment::TAG_PIPELINE_ID => pipeline_id,
Expand Down Expand Up @@ -86,6 +86,13 @@ def tags
# @type var v: untyped
v.nil? || v.strip.empty?
end

# Normalize Git references and filter sensitive data
normalize_git!(tags)
# Expand ~
expand_workspace!(tags)

tags
end

private
Expand Down Expand Up @@ -172,6 +179,30 @@ def git_commit_message
def git_commit_sha
end

def normalize_git!(tags)
if !tags[Git::TAG_BRANCH].nil? && tags[Git::TAG_BRANCH].include?("tags/")
tags[Git::TAG_TAG] = tags[Git::TAG_BRANCH]
tags.delete(Git::TAG_BRANCH)
end

tags[Git::TAG_TAG] = normalize_ref(tags[Git::TAG_TAG]) if tags[Git::TAG_TAG]
tags[Git::TAG_BRANCH] = normalize_ref(tags[Git::TAG_BRANCH]) if tags[Git::TAG_BRANCH]

if tags[Git::TAG_REPOSITORY_URL]
tags[Git::TAG_REPOSITORY_URL] = filter_sensitive_info(
tags[Git::TAG_REPOSITORY_URL]
)
end
end

def expand_workspace!(tags)
workspace_path = tags[TAG_WORKSPACE_PATH]

if !workspace_path.nil? && (workspace_path == "~" || workspace_path.start_with?("~/"))
tags[TAG_WORKSPACE_PATH] = File.expand_path(workspace_path)
end
end

def set_branch_and_tag
branch_or_tag_string = git_branch_or_tag
@branch = @tag = nil
Expand All @@ -185,14 +216,18 @@ def set_branch_and_tag
end

def normalize_ref(name)
return nil if name.nil?

refs = %r{^refs/(heads/)?}
origin = %r{^origin/}
tags = %r{^tags/}
name.gsub(refs, "").gsub(origin, "").gsub(tags, "") unless name.nil?
name.gsub(refs, "").gsub(origin, "").gsub(tags, "")
end

def filter_sensitive_info(url)
url.gsub(%r{(https?://)[^/]*@}, '\1') unless url.nil?
return nil if url.nil?

url.gsub(%r{(https?://)[^/]*@}, '\1')
end
end
end
Expand Down
Loading

0 comments on commit fd3ed19

Please sign in to comment.