From 9d1afdd4a01881199149a343aed2e96fbd3ab25f Mon Sep 17 00:00:00 2001 From: Miroslav Shubernetskiy Date: Tue, 10 Dec 2024 11:46:06 -0500 Subject: [PATCH] fix: parsing git timestamp offsets When git reports its timestamp, its localized with the provided offset and as chalk wasnt accounting for that, in some cases, commit timestamps would show up as future timestamps which is very confusing Now all timestamps are normalized with the provided offset. In addition all date reporting was changed to use consistent output format of ISO-8601. --- CHANGELOG.md | 62 ++++++++++++++- src/chalk_common.nim | 9 ++- src/configs/base_keyspecs.c4m | 97 ++++++++++++++++++++++-- src/configs/base_plugins.c4m | 12 +-- src/configs/base_report_templates.c4m | 36 +++++++++ src/configs/crashoverride.c4m | 6 ++ src/plugins/system.nim | 39 ++++------ src/plugins/vctlGit.nim | 104 +++++++++++++------------- src/run_management.nim | 10 ++- src/util.nim | 11 ++- tests/functional/test_git.py | 8 +- tests/functional/utils/git.py | 2 +- 12 files changed, 291 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 202965d1..e177de11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -127,10 +127,68 @@ ([#450](https://github.com/crashappsec/chalk/pull/450), [#453](https://github.com/crashappsec/chalk/pull/453)) +- Git time-related fields are now reported in ISO-8601 format whereas + previously it was reporting using default git format. + + Before: + + ```json + { + "DATE_AUTHORED": "Tue Dec 10 11:46:06 2024 -0500", + "DATE_COMMITTED": "Tue Dec 10 11:46:06 2024 -0500", + "DATE_TAGGED": "Tue Dec 10 11:46:06 2024 -0500" + } + ``` + + Now: + + ```json + { + "DATE_AUTHORED": "2024-12-10T16:46:06.000Z", + "DATE_COMMITTED": "2024-12-10T18:49:00.000Z", + "DATE_TAGGED": "2024-12-10T18:49:00.000Z" + } + ``` + + This also affects all host-level keys in addition to chalk-level keys: + + - `DATE_AUTHORED` + - `DATE_COMMITTED` + - `DATE_TAGGED` + - `_DATE_AUTHORED` + - `_DATE_COMMITTED` + - `_DATE_TAGGED` + + To make parsing easier, in addition to human readable `DATE_*` fields, + new `TIMESTAMP_*` fields are added which report milliseconds since + Unix epoch: + + ```json + { + "DATE_AUTHORED": "2024-12-10T16:46:06.000Z", + "DATE_COMMITTED": "2024-12-10T18:49:00.000Z", + "DATE_TAGGED": "2024-12-10T18:49:00.000Z", + "TIMESTAMP_AUTHORED": 1733849166000, + "TIMESTAMP_COMMITTED": 1733856540000 + "TIMESTAMP_TAGGED": 1733856540000 + } + ``` + + ([#458](https://github.com/crashappsec/chalk/pull/458)) + +- All datetime fields are now reported in UTC TZ whereas previously were + reported in machines local TZ + ([#458](https://github.com/crashappsec/chalk/pull/458)) + ### Fixes -- `DOCKERFILE_PATH_WITHIN_VCTL` key is no longer reported when providing Dockerfile contents - via `stdin` ([#454](https://github.com/crashappsec/chalk/pull/454)). +- `DOCKERFILE_PATH_WITHIN_VCTL` key is no longer reported when providing + Dockerfile contents via `stdin` + ([#454](https://github.com/crashappsec/chalk/pull/454)). + +- Git time-related fields report accurate timezone now. Previously + wrong commit TZ was being reported as committed in git which was not correct. + ([#458](https://github.com/crashappsec/chalk/pull/458)) ### New Features diff --git a/src/chalk_common.nim b/src/chalk_common.nim index 3107f9ce..dc1c22ba 100644 --- a/src/chalk_common.nim +++ b/src/chalk_common.nim @@ -455,13 +455,18 @@ const techStackConfig* = staticRead(techStackConfName) linguistConfig* = staticRead(linguistConfName) ioConfig* = staticRead(ioConfName) - defaultConfig* = staticRead(defCfgFname) #& commentC4mCode(ioConfig) + defaultConfig* = staticRead(defCfgFname) attestConfig* = staticRead(attestConfName) coConfig* = staticRead(coConfName) - commitID* = staticexec("git rev-parse HEAD") + commitID* = staticexec("git log -n1 --pretty=format:%H") archStr* = staticexec("uname -m") osStr* = staticexec("uname -o") stdinIndicator* = ":stdin:" + # various time formats + timesDateFormat* = "yyyy-MM-dd" + timesTimeFormat* = "HH:mm:ss'.'fff" + timesTzFormat* = "zzz" + timesIso8601Format* = timesDateFormat & "'T'" & timesTimeFormat & timesTzFormat # Make sure that ARTIFACT_TYPE fields are consistently named. I'd love # these to be const, but nim doesn't seem to be able to handle that :( diff --git a/src/configs/base_keyspecs.c4m b/src/configs/base_keyspecs.c4m index 97a2b12b..ae8f8d10 100644 --- a/src/configs/base_keyspecs.c4m +++ b/src/configs/base_keyspecs.c4m @@ -696,7 +696,22 @@ keyspec DATE_AUTHORED { since: "0.1.2" shortdoc: "Human readable timestamp of the author's commit" doc: """ -The date of the authored commit in the same format as `git log`: 3-letter week-day, 3-letter month, day of month, HH:mm:ss +The date when commit was authored as full ISO-8601 string. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. +""" +} + +keyspec TIMESTAMP_AUTHORED { + kind: ChalkTimeArtifact + type: string + standard: true + since: "0.4.15" + shortdoc: "Unix timestamp of the author's commit" + doc: """ +The date when commit was authored as ms since Unix epoch. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. """ } @@ -718,7 +733,22 @@ keyspec DATE_COMMITTED { since: "0.1.2" shortdoc: "Human readable timestamp of most recent commit" doc: """ -The date of the authored commit in the same format as `git log`: 3-letter week-day, 3-letter month, day of month, HH:mm:ss +The date when commit was created as full ISO-8601 string. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. +""" +} + +keyspec TIMESTAMP_COMMITTED { + kind: ChalkTimeArtifact + type: string + standard: true + since: "0.1.2" + shortdoc: "Unix timestamp of most recent commit" + doc: """ +The date when commit was created as ms since Unix epoch. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. """ } @@ -794,12 +824,24 @@ keyspec DATE_TAGGED { since: "0.3.0" shortdoc: "Human readable timestamp of most recent tag" doc: """ -The date of the annotated tag in the same format as `git log`: 3-letter week-day, 3-letter month, day of month, HH:mm:ss +The date when the annotated was created as full ISO-8601 string. Annotated tags are created with `git tag -a -m "comment"`. """ } +keyspec TIMESTAMP_TAGGED { + kind: ChalkTimeArtifact + type: string + standard: true + since: "0.4.15" + shortdoc: "Unix timestamp of most recent tag" + doc: """ +The date when the annotated was created in ms since Unix epoch. + +Annotated tags are created with `git tag -a -m "comment"`. +""" +} keyspec TAG_MESSAGE { kind: ChalkTimeArtifact type: string @@ -872,7 +914,22 @@ keyspec _DATE_AUTHORED { since: "0.4.0" shortdoc: "Human readable timestamp of the author's commit" doc: """ -The date of the authored commit in the same format as `git log`: 3-letter week-day, 3-letter month, day of month, HH:mm:ss +The date when commit was authored as full ISO-8601 string. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. +""" +} + +keyspec _TIMESTAMP_AUTHORED { + kind: RunTimeHost + type: string + standard: true + since: "0.4.15" + shortdoc: "Unix timestamp of the author's commit" + doc: """ +The date when commit was authored as ms since Unix epoch. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. """ } @@ -894,7 +951,22 @@ keyspec _DATE_COMMITTED { since: "0.4.0" shortdoc: "Human readable timestamp of most recent commit" doc: """ -The date of the authored commit in the same format as `git log`: 3-letter week-day, 3-letter month, day of month, HH:mm:ss +The date when commit was created as full ISO-8601 string. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. +""" +} + +keyspec _TIMESTAMP_COMMITTED { + kind: RunTimeHost + type: string + standard: true + since: "0.4.15" + shortdoc: "Unix timestamp of most recent commit" + doc: """ +The date when commit was created as ms since Unix epoch. +Note that git commit and author dates might be different. +For example `git rebase` adjusts dates. """ } @@ -976,6 +1048,19 @@ Annotated tags are created with `git tag -a -m "comment"`. """ } +keyspec _TIMESTAMP_TAGGED { + kind: RunTimeHost + type: string + standard: true + since: "0.4.15" + shortdoc: "Unix timestamp of most recent tag" + doc: """ +The date when the annotated was created in ms since Unix epoch. + +Annotated tags are created with `git tag -a -m "comment"`. +""" +} + keyspec _TAG_MESSAGE { kind: RunTimeHost type: string @@ -5891,7 +5976,7 @@ keyspec _CHALK_RUN_TIME { doc: """ Calculates the amount of time between the start of a chalk executable and when a report is generated. It's an integer with resolution of -1/1000000th of a second. +1e-6 of a second (millisecond). """ } diff --git a/src/configs/base_plugins.c4m b/src/configs/base_plugins.c4m index 8dfabfb3..1ad2776e 100644 --- a/src/configs/base_plugins.c4m +++ b/src/configs/base_plugins.c4m @@ -91,15 +91,15 @@ plugin vctl_git { enabled: true pre_chalk_keys: ["COMMIT_ID", "COMMIT_SIGNED", "BRANCH", "ORIGIN_URI", - "AUTHOR", "DATE_AUTHORED", - "COMMITTER", "DATE_COMMITTED", "COMMIT_MESSAGE", - "TAG", "TAG_SIGNED", "TAGGER", "DATE_TAGGED", + "AUTHOR", "DATE_AUTHORED", "TIMESTAMP_AUTHORED", + "COMMITTER", "DATE_COMMITTED", "TIMESTAMP_COMMITTED", "COMMIT_MESSAGE", + "TAG", "TAG_SIGNED", "TAGGER", "DATE_TAGGED", "TIMESTAMP_TAGGED", "TAG_MESSAGE", "VCS_DIR_WHEN_CHALKED"] post_run_keys: ["_COMMIT_ID", "_COMMIT_SIGNED", "_BRANCH", "_ORIGIN_URI", - "_AUTHOR", "_DATE_AUTHORED", - "_COMMITTER", "_DATE_COMMITTED", "_COMMIT_MESSAGE", - "_TAG", "_TAG_SIGNED", "_TAGGER", "_DATE_TAGGED", + "_AUTHOR", "_DATE_AUTHORED", "_TIMESTAMP_AUTHORED", + "_COMMITTER", "_DATE_COMMITTED", "_TIMESTAMP_COMMITTED", "_COMMIT_MESSAGE", + "_TAG", "_TAG_SIGNED", "_TAGGER", "_DATE_TAGGED", "_TIMESTAMP_TAGGED", "_TAG_MESSAGE"] doc: """ This plugin collects version control information from .git directories diff --git a/src/configs/base_report_templates.c4m b/src/configs/base_report_templates.c4m index 1e4c8ad2..0382d92c 100644 --- a/src/configs/base_report_templates.c4m +++ b/src/configs/base_report_templates.c4m @@ -59,11 +59,14 @@ report and subtract from it. key.COMMIT_SIGNED.use = true key.AUTHOR.use = true key.DATE_AUTHORED.use = true + key.TIMESTAMP_AUTHORED.use = true key.COMMITTER.use = true key.DATE_COMMITTED.use = true + key.TIMESTAMP_COMMITTED.use = true key.COMMIT_MESSAGE.use = true key.TAGGER.use = true key.DATE_TAGGED.use = true + key.TIMESTAMP_TAGGED.use = true key.TAG_MESSAGE.use = true key._ORIGIN_URI.use = true key._BRANCH.use = true @@ -73,11 +76,14 @@ report and subtract from it. key._COMMIT_SIGNED.use = true key._AUTHOR.use = true key._DATE_AUTHORED.use = true + key._TIMESTAMP_AUTHORED.use = true key._COMMITTER.use = true key._DATE_COMMITTED.use = true + key._TIMESTAMP_COMMITTED.use = true key._COMMIT_MESSAGE.use = true key._TAGGER.use = true key._DATE_TAGGED.use = true + key._TIMESTAMP_TAGGED.use = true key._TAG_MESSAGE.use = true key.ARTIFACT_VERSION.use = true key.STORE_URI.use = true @@ -650,11 +656,14 @@ doc: """ key.COMMIT_SIGNED.use = true key.AUTHOR.use = true key.DATE_AUTHORED.use = true + key.TIMESTAMP_AUTHORED.use = true key.COMMITTER.use = true key.DATE_COMMITTED.use = true + key.TIMESTAMP_COMMITTED.use = true key.COMMIT_MESSAGE.use = true key.TAGGER.use = true key.DATE_TAGGED.use = true + key.TIMESTAMP_TAGGED.use = true key.TAG_MESSAGE.use = true key._ORIGIN_URI.use = true key._BRANCH.use = true @@ -664,11 +673,14 @@ doc: """ key._COMMIT_SIGNED.use = true key._AUTHOR.use = true key._DATE_AUTHORED.use = true + key._TIMESTAMP_AUTHORED.use = true key._COMMITTER.use = true key._DATE_COMMITTED.use = true + key._TIMESTAMP_COMMITTED.use = true key._COMMIT_MESSAGE.use = true key._TAGGER.use = true key._DATE_TAGGED.use = true + key._TIMESTAMP_TAGGED.use = true key._TAG_MESSAGE.use = true key.ARTIFACT_VERSION.use = true key.STORE_URI.use = true @@ -1124,11 +1136,14 @@ container. key.COMMIT_SIGNED.use = true key.AUTHOR.use = true key.DATE_AUTHORED.use = true + key.TIMESTAMP_AUTHORED.use = true key.COMMITTER.use = true key.DATE_COMMITTED.use = true + key.TIMESTAMP_COMMITTED.use = true key.COMMIT_MESSAGE.use = true key.TAGGER.use = true key.DATE_TAGGED.use = true + key.TIMESTAMP_TAGGED.use = true key.TAG_MESSAGE.use = true key._ORIGIN_URI.use = true key._BRANCH.use = true @@ -1138,11 +1153,14 @@ container. key._COMMIT_SIGNED.use = true key._AUTHOR.use = true key._DATE_AUTHORED.use = true + key._TIMESTAMP_AUTHORED.use = true key._COMMITTER.use = true key._DATE_COMMITTED.use = true + key._TIMESTAMP_COMMITTED.use = true key._COMMIT_MESSAGE.use = true key._TAGGER.use = true key._DATE_TAGGED.use = true + key._TIMESTAMP_TAGGED.use = true key._TAG_MESSAGE.use = true key.ARTIFACT_VERSION.use = true key.STORE_URI.use = true @@ -1598,11 +1616,14 @@ and keep the run-time key. key.COMMIT_SIGNED.use = true key.AUTHOR.use = true key.DATE_AUTHORED.use = true + key.TIMESTAMP_AUTHORED.use = true key.COMMITTER.use = true key.DATE_COMMITTED.use = true + key.TIMESTAMP_COMMITTED.use = true key.COMMIT_MESSAGE.use = true key.TAGGER.use = true key.DATE_TAGGED.use = true + key.TIMESTAMP_TAGGED.use = true key.TAG_MESSAGE.use = true key._ORIGIN_URI.use = true key._BRANCH.use = true @@ -1612,11 +1633,14 @@ and keep the run-time key. key._COMMIT_SIGNED.use = true key._AUTHOR.use = true key._DATE_AUTHORED.use = true + key._TIMESTAMP_AUTHORED.use = true key._COMMITTER.use = true key._DATE_COMMITTED.use = true + key._TIMESTAMP_COMMITTED.use = true key._COMMIT_MESSAGE.use = true key._TAGGER.use = true key._DATE_TAGGED.use = true + key._TIMESTAMP_TAGGED.use = true key._TAG_MESSAGE.use = true key.ARTIFACT_VERSION.use = true key.STORE_URI.use = true @@ -1994,11 +2018,14 @@ running insert commands. key.COMMIT_SIGNED.use = true key.AUTHOR.use = true key.DATE_AUTHORED.use = true + key.TIMESTAMP_AUTHORED.use = true key.COMMITTER.use = true key.DATE_COMMITTED.use = true + key.TIMESTAMP_COMMITTED.use = true key.COMMIT_MESSAGE.use = true key.TAGGER.use = true key.DATE_TAGGED.use = true + key.TIMESTAMP_TAGGED.use = true key.TAG_MESSAGE.use = true key._ORIGIN_URI.use = true key._BRANCH.use = true @@ -2008,11 +2035,14 @@ running insert commands. key._COMMIT_SIGNED.use = true key._AUTHOR.use = true key._DATE_AUTHORED.use = true + key._TIMESTAMP_AUTHORED.use = true key._COMMITTER.use = true key._DATE_COMMITTED.use = true + key._TIMESTAMP_COMMITTED.use = true key._COMMIT_MESSAGE.use = true key._TAGGER.use = true key._DATE_TAGGED.use = true + key._TIMESTAMP_TAGGED.use = true key._TAG_MESSAGE.use = true key.EMBEDDED_CHALK.use = true key.ERR_INFO.use = true @@ -2084,11 +2114,14 @@ running commands that do NOT insert chalk marks. key.COMMIT_SIGNED.use = true key.AUTHOR.use = true key.DATE_AUTHORED.use = true + key.TIMESTAMP_AUTHORED.use = true key.COMMITTER.use = true key.DATE_COMMITTED.use = true + key.TIMESTAMP_COMMITTED.use = true key.COMMIT_MESSAGE.use = true key.TAGGER.use = true key.DATE_TAGGED.use = true + key.TIMESTAMP_TAGGED.use = true key.TAG_MESSAGE.use = true key._ORIGIN_URI.use = true key._BRANCH.use = true @@ -2098,11 +2131,14 @@ running commands that do NOT insert chalk marks. key._COMMIT_SIGNED.use = true key._AUTHOR.use = true key._DATE_AUTHORED.use = true + key._TIMESTAMP_AUTHORED.use = true key._COMMITTER.use = true key._DATE_COMMITTED.use = true + key._TIMESTAMP_COMMITTED.use = true key._COMMIT_MESSAGE.use = true key._TAGGER.use = true key._DATE_TAGGED.use = true + key._TIMESTAMP_TAGGED.use = true key._TAG_MESSAGE.use = true key.EMBEDDED_CHALK.use = true key.ERR_INFO.use = true diff --git a/src/configs/crashoverride.c4m b/src/configs/crashoverride.c4m index c0ed8f75..5da4bc48 100644 --- a/src/configs/crashoverride.c4m +++ b/src/configs/crashoverride.c4m @@ -175,11 +175,14 @@ This is mostly a copy of insert template however all keys are immutable. ~key.COMMIT_SIGNED.use = true ~key.AUTHOR.use = true ~key.DATE_AUTHORED.use = true + ~key.TIMESTAMP_AUTHORED.use = true ~key.COMMITTER.use = true ~key.DATE_COMMITTED.use = true + ~key.TIMESTAMP_COMMITTED.use = true ~key.COMMIT_MESSAGE.use = true ~key.TAGGER.use = true ~key.DATE_TAGGED.use = true + ~key.TIMESTAMP_TAGGED.use = true ~key.TAG_MESSAGE.use = true ~key._ORIGIN_URI.use = true ~key._BRANCH.use = true @@ -189,11 +192,14 @@ This is mostly a copy of insert template however all keys are immutable. ~key._COMMIT_SIGNED.use = true ~key._AUTHOR.use = true ~key._DATE_AUTHORED.use = true + ~key._TIMESTAMP_AUTHORED.use = true ~key._COMMITTER.use = true ~key._DATE_COMMITTED.use = true + ~key._TIMESTAMP_COMMITTED.use = true ~key._COMMIT_MESSAGE.use = true ~key._TAGGER.use = true ~key._DATE_TAGGED.use = true + ~key._TIMESTAMP_TAGGED.use = true ~key._TAG_MESSAGE.use = true ~key.ARTIFACT_VERSION.use = true ~key.STORE_URI.use = true diff --git a/src/plugins/system.nim b/src/plugins/system.nim index 9562477c..a4192be7 100644 --- a/src/plugins/system.nim +++ b/src/plugins/system.nim @@ -11,7 +11,7 @@ when defined(posix): import std/posix_utils -import std/[monotimes, nativesockets, sequtils, times] +import std/[nativesockets, sequtils, times, monotimes] import ".."/[config, plugin_api, normalize, chalkjson, attestation_api, util] @@ -79,9 +79,9 @@ proc sysGetChalkTimeArtifactInfo*(self: Plugin, obj: ChalkObj): if ResourceImage in obj.resourceType: if "TIMESTAMP_WHEN_CHALKED" notin obj.collectedData: # image is immutable so cannot overwrite timestamp from original build - result["TIMESTAMP_WHEN_CHALKED"] = pack(unixTimeInMS()) + result["TIMESTAMP_WHEN_CHALKED"] = pack(startTime.toUnixInMs()) else: - result["TIMESTAMP_WHEN_CHALKED"] = pack(unixTimeInMS()) + result["TIMESTAMP_WHEN_CHALKED"] = pack(startTime.toUnixInMs()) if isSubscribedKey("PRE_CHALK_HASH") and obj.fsRef != "": withFilesTream(obj.fsRef, mode = fmRead, strict = true): @@ -166,8 +166,6 @@ proc sysGetRunTimeArtifactInfo*(self: Plugin, obj: ChalkObj, insert: bool): result["_OP_ARTIFACT_REPORT_KEYS"] = pack(reportKeys) var - instant = epochTime() - timestamp = instant.fromUnixFloat() envdict = Con4mDict[string, string]() cachedSearchPath = newSeq[string]() @@ -178,12 +176,6 @@ proc clearCallback(self: Plugin) {.cdecl.} = when defined(posix): let uinfo = uname() -template getDate(): string = timestamp.format("yyyy-MM-dd") -template getTime(): string = timestamp.format("HH:mm:ss") & "." & - timestamp.format("fff") -template getOffset(): string = timestamp.format("zzz") -template getDateTime(): string = getDate() & "T" & getTime() & getOffset() - proc getEnvDict(): Box = if len(envdict) == 0: envdict = Con4mDict[string, string]() @@ -230,11 +222,11 @@ proc sysGetRunTimeHostInfo*(self: Plugin, objs: seq[ChalkObj]): result.setIfNeeded("_OP_ARGV", @[getMyAppPath()] & commandLineParams()) result.setIfNeeded("_OP_HOSTNAME", getHostName()) result.setIfNeeded("_OP_UNMARKED_COUNT", len(getUnmarked())) - result.setIfNeeded("_TIMESTAMP", uint64(instant * 1000.0)) - result.setIfNeeded("_DATE", pack(getDate())) - result.setIfNeeded("_TIME", pack(getTime())) - result.setIfNeeded("_TZ_OFFSET", pack(getOffset())) - result.setIfNeeded("_DATETIME", pack(getDateTime())) + result.setIfNeeded("_TIMESTAMP", startTime.toUnixInMs()) + result.setIfNeeded("_DATE", startTime.utc.format(timesDateFormat)) + result.setIfNeeded("_TIME", startTime.utc.format(timesTimeFormat)) + result.setIfNeeded("_TZ_OFFSET", startTime.utc.format(timesTzFormat)) + result.setIfNeeded("_DATETIME", startTime.utc.format(timesIso8601Format)) if isSubscribedKey("_ENV"): result["_ENV"] = getEnvDict() @@ -264,10 +256,10 @@ proc sysGetChalkTimeHostInfo*(self: Plugin): ChalkDict {.cdecl.} = result.setIfNeeded("INJECTOR_VERSION", getChalkExeVersion()) result.setIfNeeded("INJECTOR_COMMIT_ID", getChalkCommitId()) result.setIfNeeded("INJECTOR_ENV", getEnvDict()) - result.setIfNeeded("DATE_CHALKED", pack(getDate())) - result.setIfNeeded("TIME_CHALKED", pack(getTime())) - result.setIfNeeded("TZ_OFFSET_WHEN_CHALKED", pack(getOffset())) - result.setIfNeeded("DATETIME_WHEN_CHALKED", pack(getDateTime())) + result.setIfNeeded("DATE_CHALKED", startTime.utc.format(timesDateFormat)) + result.setIfNeeded("TIME_CHALKED", startTime.utc.format(timesTimeFormat)) + result.setIfNeeded("TZ_OFFSET_WHEN_CHALKED", startTime.utc.format(timesTzFormat)) + result.setIfNeeded("DATETIME_WHEN_CHALKED", startTime.utc.format(timesIso8601Format)) result.setIfNeeded("PLATFORM_WHEN_CHALKED", getChalkPlatform()) result.setIfNeeded("PUBLIC_IPV4_ADDR_WHEN_CHALKED", pack(getMyIpV4Addr())) @@ -325,11 +317,10 @@ proc metsysGetRunTimeHostInfo(self: Plugin, objs: seq[ChalkObj]): result.setIfNeeded("_CHALK_EXTERNAL_ACTION_AUDIT", externalActions) if isSubscribedKey("_CHALK_RUN_TIME"): - # startTime lives in runManagement. let - diff = getMonoTime().ticks() - startTime - inMs = diff div 1000 # It's in nanosec, convert to 1/1000000th of a sec - + monoEndTime = getMonoTime() + diff = monoEndTime - monoStartTime + inMs = diff.inMilliseconds() result["_CHALK_RUN_TIME"] = pack(inMs) proc loadSystem*() = diff --git a/src/plugins/vctlGit.nim b/src/plugins/vctlGit.nim index 79b9cce0..022ad293 100644 --- a/src/plugins/vctlGit.nim +++ b/src/plugins/vctlGit.nim @@ -42,7 +42,6 @@ const gitPackRefs = "packed-refs" gitPack = gitObjects.joinPath("pack") gitPackExt = ".pack" - gitTimeFmt = "ddd MMM dd HH:mm:ss YYYY" # https://git-scm.com/docs/pack-format gitObjCommit = 1 gitObjTag = 4 @@ -54,13 +53,16 @@ const keyBranch = "BRANCH" keyAuthor = "AUTHOR" keyAuthorDate = "DATE_AUTHORED" + keyAuthorTime = "TIMESTAMP_AUTHORED" keyCommitter = "COMMITTER" keyCommitDate = "DATE_COMMITTED" + keyCommitTime = "TIMESTAMP_COMMITTED" keyCommitMessage = "COMMIT_MESSAGE" keyLatestTag = "TAG" keyTagSigned = "TAG_SIGNED" keyTagger = "TAGGER" keyTaggedDate = "DATE_TAGGED" + keyTaggedTime = "TIMESTAMP_TAGGED" keyTagMessage = "TAG_MESSAGE" type @@ -247,8 +249,8 @@ type commitId: string tagCommitId: string tagger: string - unixTime: int - date: string + date: DateTime + timestamp: int64 # needed for sorting signed: bool message: string @@ -261,9 +263,9 @@ type commitId: string signed: bool author: string - authorDate: string + authorDate: DateTime committer: string - commitDate: string + commitDate: DateTime message: string GitInfo = ref object of RootRef @@ -281,11 +283,15 @@ proc isAnnotated(self: GitTag): bool = proc getUint32BE(data: string, whence: SomeInteger=0): uint32 = result = ntohl(cast[ptr [uint32]](addr data[whence])[]) -template parseTime(line: string): int = - parseInt(line.split()[^2]) +proc extractPerson(line: string): string = + let parts = line.rsplit(maxsplit = 2) + return parts[0] -template formatCommitObjectTime(line: string): string = - fromUnix(parseTime(line)).format(gitTimeFmt) & " " & line.split()[^1] +proc parseGitDateTime(line: string): DateTime = + let + parts = line.rsplit(maxsplit = 3) + seconds = parseInt(parts[^2]) + result = fromUnix(seconds).utc proc readPackedObject(path: string, offset: uint64): string = ## read packaged git object @@ -517,15 +523,18 @@ proc loadRef(info: RepoInfo, gitRef: string): string = result = commitId proc loadAuthor(info: RepoInfo) = - let fields = info.loadObject(info.commitId) + let + fields = info.loadObject(info.commitId) + author = fields.getOrDefault(gitAuthor, "") + committer = fields.getOrDefault(gitCommitter, "") # this makes only the same assumptions as git itself: - # https://github.com/git/git/blob/master/commit.c#L97 ddcb8fd - info.author = fields.getOrDefault(gitAuthor, "") + # https://github.com/git/git/blob/master/commit.c#L97 + info.author = author.extractPerson() if info.author != "": - info.authorDate = formatCommitObjectTime(info.author) - info.committer = fields.getOrDefault(gitCommitter, "") + info.authorDate = parseGitDateTime(author) + info.committer = committer.extractPerson() if info.committer != "": - info.commitDate = formatCommitObjectTime(info.committer) + info.commitDate = parseGitDateTime(committer) info.message = fields.getOrDefault(gitMessage, "") info.signed = gitSign in fields @@ -548,17 +557,16 @@ proc loadTag(info: RepoInfo, tag: string, tagCommit: string) = # we found annotated commit pointing to current commit if fields[gitTag] == tag and fields[gitObject] == info.commitId: let - tagger = fields[gitTagger] - unixTime = parseTime(tagger) - date = formatCommitObjectTime(tagger) - signed = gitSign in fields - message = fields.getOrDefault(gitMessage) + tagger = fields[gitTagger] + date = parseGitDateTime(tagger) + signed = gitSign in fields + message = fields.getOrDefault(gitMessage) info.tags[tag] = GitTag(name: tag, commitId: fields[gitObject], tagCommitId: tagCommit, - tagger: tagger, - unixTime: unixTime, + tagger: tagger.extractPerson(), date: date, + timestamp: date.toUnixInMs(), signed: signed, message: message) trace("annotated tag: " & tag) @@ -589,7 +597,7 @@ proc loadTags(info: RepoInfo) = tagCommit = tagCommit) if len(info.tags) > 0: - let sortedTags = info.tags.values().toSeq().sortedByIt((it.unixTime, it.name)) + let sortedTags = info.tags.values().toSeq().sortedByIt((it.timestamp, it.name)) info.latestTag = sortedTags[^1] trace("latest tag: " & info.latestTag.name) @@ -753,37 +761,31 @@ proc findAndLoad(plugin: GitInfo, path: string) = plugin.vcsDirs[vcsDir] = info -proc pack(tag: GitTag): ChalkDict = - new result - if tag.tagger != "": - result[keyTagger] = pack(tag.tagger) - result[keyTaggedDate] = pack(formatCommitObjectTime(tag.tagger)) - -proc pack(tags: Table[string, GitTag]): ChalkDict = - new result - for name, tag in tags: - result[name] = pack(tag.pack()) - proc setVcsKeys(chalkDict: ChalkDict, info: RepoInfo, prefix = "") = if prefix == "": - chalkDict.setIfNeeded(prefix & keyVcsDir, info.vcsDir.splitPath().head) - - chalkDict.setIfNeeded(prefix & keyOrigin, info.origin) - chalkDict.setIfNeeded(prefix & keyCommit, info.commitId) - chalkDict.setIfNeeded(prefix & keyCommitSigned, info.signed) - chalkDict.setIfNeeded(prefix & keyBranch, info.branch) - chalkDict.setIfNeeded(prefix & keyAuthor, info.author) - chalkDict.setIfNeeded(prefix & keyAuthorDate, info.authorDate) - chalkDict.setIfNeeded(prefix & keyCommitter, info.committer) - chalkDict.setIfNeeded(prefix & keyCommitDate, info.commitDate) - chalkDict.setIfNeeded(prefix & keyCommitMessage, info.message) - + chalkDict.setIfNeeded(prefix & keyVcsDir, info.vcsDir.splitPath().head) + + chalkDict.setIfNeeded(prefix & keyOrigin, info.origin) + chalkDict.setIfNeeded(prefix & keyCommit, info.commitId) + chalkDict.setIfNeeded(prefix & keyCommitSigned, info.signed) + chalkDict.setIfNeeded(prefix & keyBranch, info.branch) + chalkDict.setIfNeeded(prefix & keyAuthor, info.author) + chalkDict.setIfNeeded(prefix & keyCommitter, info.committer) + chalkDict.setIfNeeded(prefix & keyCommitMessage, info.message) + if info.author != "": + chalkDict.setIfNeeded(prefix & keyAuthorDate, info.authorDate.utc.format(timesIso8601Format)) + chalkDict.setIfNeeded(prefix & keyAuthorTime, info.authorDate.toUnixInMs()) + if info.committer != "": + chalkDict.setIfNeeded(prefix & keyCommitDate, info.commitDate.utc.format(timesIso8601Format)) + chalkDict.setIfNeeded(prefix & keyCommitTime, info.commitDate.toUnixInMs()) if info.latestTag != nil: - chalkDict.setIfNeeded(prefix & keyLatestTag, info.latestTag.name) - chalkDict.setIfNeeded(prefix & keyTagger, info.latestTag.tagger) - chalkDict.setIfNeeded(prefix & keyTaggedDate, info.latestTag.date) - chalkDict.setIfNeeded(prefix & keyTagSigned, info.latestTag.signed) - chalkDict.setIfNeeded(prefix & keyTagMessage, info.latestTag.message) + chalkDict.setIfNeeded(prefix & keyLatestTag, info.latestTag.name) + chalkDict.setIfNeeded(prefix & keyTagger, info.latestTag.tagger) + if info.latestTag.tagger != "": + chalkDict.setIfNeeded(prefix & keyTaggedDate, info.latestTag.date.utc.format(timesIso8601Format)) + chalkDict.setIfNeeded(prefix & keyTaggedTime, info.latestTag.date.toUnixInMs()) + chalkDict.setIfNeeded(prefix & keyTagSigned, info.latestTag.signed) + chalkDict.setIfNeeded(prefix & keyTagMessage, info.latestTag.message) proc isInRepo(obj: ChalkObj, repo: string): bool = if obj.fsRef == "": diff --git a/src/run_management.nim b/src/run_management.nim index 1b90318d..fe2dd18c 100644 --- a/src/run_management.nim +++ b/src/run_management.nim @@ -9,13 +9,14 @@ ## setting, status stuff, and the core scan state ("collection ## contexts"), that the subscan module pushes and pops. -import std/[posix, monotimes, enumerate] +import std/[posix, monotimes, enumerate, times] import "."/chalk_common export chalk_common var - ctxStack = @[CollectionCtx()] - startTime* = getMonoTime().ticks() + ctxStack = @[CollectionCtx()] + startTime* = getTime().utc # gives absolute wall time + monoStartTime* = getMonoTime() # used for computing diffs proc getChalkScope*(): AttrScope = con4mRuntime.configState.attrs @@ -75,7 +76,8 @@ proc inSubscan*(): bool = return len(ctxStack) > 1 proc clearReportingState*() = - startTime = getMonoTime().ticks() + startTime = getTime().utc + monoStartTime = getMonoTime() ctxStack = @[CollectionCtx()] hostInfo = ChalkDict() subscribedKeys = Table[string, bool]() diff --git a/src/util.nim b/src/util.nim index 912ea75d..fe1c02be 100644 --- a/src/util.nim +++ b/src/util.nim @@ -8,7 +8,7 @@ ## This is for any common code for system stuff, such as executing ## code. -import std/[httpcore, tempfiles, posix, monotimes, parseutils, exitprocs, sets] +import std/[httpcore, tempfiles, posix, parseutils, exitprocs, sets, times, monotimes] import pkg/[nimutils/managedtmp] import "."/[config, subscan, fd_cache] export fd_cache @@ -79,10 +79,9 @@ proc reportTmpFileExitState*(files, dirs, errs: seq[string]) = for item in files & dirs: error(item) + let monoEndTime = getMonoTime() if attrGet[bool]("report_total_time"): - echo "Total run time: " & $(int(getMonoTime().ticks() - startTime) / - 1000000000) & - " seconds" + echo("Total run time: " & $(monoEndTime - monoStartTime)) proc canOpenFile*(path: string, mode: FileMode = FileMode.fmRead): bool = var canOpen = false @@ -646,3 +645,7 @@ proc `+`*[T](a, b: OrderedSet[T]): OrderedSet[T] = result.incl(i) for i in b: result.incl(i) + +proc toUnixInMs*(t: DateTime): int64 = + let epoch = fromUnix(0).utc + return (t - epoch).inMilliseconds() diff --git a/tests/functional/test_git.py b/tests/functional/test_git.py index c1bbacba..ae11f823 100644 --- a/tests/functional/test_git.py +++ b/tests/functional/test_git.py @@ -73,20 +73,18 @@ def test_repo( git.symbolic_ref(f"refs/tags/foo/{random_hex}-2") artifact = copy_files[0] result = chalk_copy.insert(artifact) - author = re.compile(rf"^{git.author} \d+ [+-]\d+$") - committer = re.compile(rf"^{git.committer} \d+ [+-]\d+$") assert result.mark.has( BRANCH="foo/bar" if not symbolic_ref else MISSING, COMMIT_ID=ANY, COMMIT_SIGNED=sign, - AUTHOR=author, + AUTHOR=git.author, DATE_AUTHORED=DATE_FORMAT, - COMMITTER=committer, + COMMITTER=git.committer, DATE_COMMITTED=DATE_FORMAT, COMMIT_MESSAGE=commit_message if not empty else MISSING, TAG=f"foo/{random_hex}-2", TAG_SIGNED=sign, - TAGGER=committer if (sign or annotate) else MISSING, + TAGGER=git.committer if (sign or annotate) else MISSING, DATE_TAGGED=DATE_FORMAT if (sign or annotate) else MISSING, TAG_MESSAGE=(tag_message if (sign or annotate) and not empty else MISSING), ORIGIN_URI=remote or "local", diff --git a/tests/functional/utils/git.py b/tests/functional/utils/git.py index 15233b47..dba1ff4c 100644 --- a/tests/functional/utils/git.py +++ b/tests/functional/utils/git.py @@ -15,7 +15,7 @@ logger = get_logger() -DATE_FORMAT = re.compile(r"^\w+ \w+ \d+ \d+:\d+:\d+ \d+ [+-]\d+$") +DATE_FORMAT = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}([+-]\d{4}|Z)$") class Git: