diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index be94a40d4..7308d8d6b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,3 +21,4 @@ serverless @Datadog/serverless sidecar @Datadog/libdatadog-php @Datadog/libdatadog-apm sidecar-ffi @Datadog/libdatadog-php @Datadog/libdatadog-apm data-pipeline*/ @Datadog/libdatadog-apm +ddsketch @Datadog/libdatadog-apm @Datadog/libdatadog-telemetry diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2213df7f3..6f813e7d4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,9 @@ variables: value: "main" description: "downstream jobs are triggered on this branch" +include: + - local: .gitlab/benchmarks.yml + trigger_internal_build: variables: LIBDATADOG_COMMIT_BRANCH: $CI_COMMIT_BRANCH @@ -21,7 +24,7 @@ trigger_internal_build: report_failure: tags: ["arch:amd64"] when: on_failure - needs: [trigger_internal_build] + needs: [trigger_internal_build, benchmarks] # allow_failure: true prevents the job from showing up in github statuses (because it's filtered by gitsync) allow_failure: true script: diff --git a/.gitlab/benchmarks.yml b/.gitlab/benchmarks.yml new file mode 100644 index 000000000..b09e71bba --- /dev/null +++ b/.gitlab/benchmarks.yml @@ -0,0 +1,34 @@ +variables: + BASE_CI_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:libdatadog-benchmarks + # The Dockerfile to this image is located at: + # https://github.com/DataDog/benchmarking-platform/tree/libdatadog/benchmarks + +benchmarks: + tags: ["runner:apm-k8s-tweaked-metal"] + needs: [] + image: + name: $BASE_CI_IMAGE + interruptible: true + timeout: 1h + script: + - export ARTIFACTS_DIR="$(pwd)/reports" && (mkdir "${ARTIFACTS_DIR}" || :) + - git clone --branch libdatadog/benchmarks https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform /platform && cd /platform + - ./steps/capture-hardware-software-info.sh + - ./steps/run-benchmarks.sh + - ./steps/analyze-results.sh + - "./steps/upload-results-to-s3.sh || :" + - "./steps/post-pr-comment.sh || :" + artifacts: + name: "reports" + paths: + - reports/ + expire_in: 3 months + variables: + UPSTREAM_PROJECT_ID: $CI_PROJECT_ID # The ID of the current project. This ID is unique across all projects on the GitLab instance. + UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME # libdatadog + UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built. + UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for. + UPSTREAM_REPO_URL: "https://github.com/DataDog/libdatadog" # The HTTP URL to the project's repository. + + KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: libdatadog + FF_USE_LEGACY_KUBERNETES_EXECUTION_STRATEGY: "true" diff --git a/Cargo.lock b/Cargo.lock index d2fa429d7..1b0bca870 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -465,6 +465,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "basic-cookies" version = "0.1.5" @@ -967,6 +973,7 @@ dependencies = [ "ciborium", "clap 4.4.18", "criterion-plot", + "csv", "is-terminal", "itertools 0.10.5", "num-traits", @@ -1032,6 +1039,27 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "current_platform" version = "0.2.0" @@ -1147,6 +1175,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "datadog-ddsketch" +version = "10.0.0" +dependencies = [ + "prost 0.11.9", + "prost-build", + "protoc-bin-vendored", +] + [[package]] name = "datadog-ipc" version = "0.1.0" @@ -1429,7 +1466,9 @@ dependencies = [ "log", "prost 0.11.9", "rand", + "rmp", "rmp-serde", + "rmpv", "serde", "serde_json", "tokio", @@ -1476,6 +1515,8 @@ name = "ddtelemetry" version = "10.0.0" dependencies = [ "anyhow", + "base64 0.22.1", + "datadog-ddsketch", "ddcommon", "futures", "hashbrown 0.12.3", @@ -2013,7 +2054,7 @@ version = "7.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" dependencies = [ - "base64", + "base64 0.21.7", "byteorder", "flate2", "nom", @@ -2106,7 +2147,7 @@ dependencies = [ "async-object-pool", "async-std", "async-trait", - "base64", + "base64 0.21.7", "basic-cookies", "crossbeam-utils", "form_urlencoded", @@ -3513,9 +3554,9 @@ dependencies = [ [[package]] name = "rmp" -version = "0.8.12" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9860a6cc38ed1da53456442089b4dfa35e7cedaa326df63017af88385e6b20" +checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" dependencies = [ "byteorder", "num-traits", @@ -3533,6 +3574,16 @@ dependencies = [ "serde", ] +[[package]] +name = "rmpv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58450723cd9ee93273ce44a20b6ec4efe17f8ed2e3631474387bfdecf18bb2a9" +dependencies = [ + "num-traits", + "rmp", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -3602,7 +3653,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] @@ -3760,7 +3811,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee80b0e361bbf88fd2f6e242ccd19cfda072cb0faa6ae694ecee08199938569a" dependencies = [ - "base64", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -4450,7 +4501,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.21.7", "bytes", "futures-core", "futures-util", diff --git a/Cargo.toml b/Cargo.toml index 0df61f1d5..6833f61a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,8 @@ members = [ "serverless", "bin_tests", "data-pipeline", - "data-pipeline-ffi" + "data-pipeline-ffi", + "ddsketch", ] # https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2 resolver = "2" diff --git a/LICENSE-3rdparty.yml b/LICENSE-3rdparty.yml index 33120e465..1d836f45b 100644 --- a/LICENSE-3rdparty.yml +++ b/LICENSE-3rdparty.yml @@ -1,4 +1,4 @@ -root_name: datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-profiling, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddcommon-ffi, build_common, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, tools, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent +root_name: datadog-alloc, datadog-crashtracker, ddcommon, ddtelemetry, datadog-ddsketch, datadog-profiling, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddcommon-ffi, build_common, ddtelemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, tools, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent third_party_libraries: - package_name: addr2line package_version: 0.21.0 @@ -2658,6 +2658,36 @@ third_party_libraries: THE SOFTWARE. - license: Apache-2.0 text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" +- package_name: base64 + package_version: 0.22.1 + repository: https://github.com/marshallpierce/rust-base64 + license: MIT OR Apache-2.0 + licenses: + - license: MIT + text: | + The MIT License (MIT) + + Copyright (c) 2015 Alice Maz + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + - license: Apache-2.0 + text: " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright [yyyy] [name of copyright owner]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n\thttp://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" - package_name: basic-cookies package_version: 0.1.5 repository: https://github.com/drjokepu/basic-cookies @@ -17020,19 +17050,19 @@ third_party_libraries: - package_name: ring package_version: 0.16.20 repository: https://github.com/briansmith/ring - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/ring-0.16.20/LICENSE) + license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/ring-0.16.20/LICENSE) licenses: - - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/ring-0.16.20/LICENSE) + - license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/ring-0.16.20/LICENSE) text: "Note that it is easy for this file to get out of sync with the licenses in the\nsource code files. It's recommended to compare the licenses in the source code\nwith what's mentioned here.\n\n*ring* is derived from BoringSSL, so the licensing situation in *ring* is\nsimilar to BoringSSL.\n\n*ring* uses an ISC-style license like BoringSSL for code in new files,\nincluding in particular all the Rust code:\n\n Copyright 2015-2016 Brian Smith.\n\n Permission to use, copy, modify, and/or distribute this software for any\n purpose with or without fee is hereby granted, provided that the above\n copyright notice and this permission notice appear in all copies.\n\n THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHORS DISCLAIM ALL WARRANTIES\n WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY\n SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION\n OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN\n CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n\nBoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL\nlicensing. Files that are completely new have a Google copyright and an ISC\nlicense. This license is reproduced at the bottom of this file.\n\nContributors to BoringSSL are required to follow the CLA rules for Chromium:\nhttps://cla.developers.google.com/clas\n\nFiles in third_party/ have their own licenses, as described therein. The MIT\nlicense, for third_party/fiat, which, unlike other third_party directories, is\ncompiled into non-test libraries, is included below.\n\nThe OpenSSL toolkit stays under a dual license, i.e. both the conditions of the\nOpenSSL License and the original SSLeay license apply to the toolkit. See below\nfor the actual license texts. Actually both licenses are BSD-style Open Source\nlicenses. In case of any license issues related to OpenSSL please contact\nopenssl-core@openssl.org.\n\nThe following are Google-internal bug numbers where explicit permission from\nsome authors is recorded for use of their work:\n 27287199\n 27287880\n 27287883\n\n OpenSSL License\n ---------------\n\n/* ====================================================================\n * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n *\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer. \n *\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in\n * the documentation and/or other materials provided with the\n * distribution.\n *\n * 3. All advertising materials mentioning features or use of this\n * software must display the following acknowledgment:\n * \"This product includes software developed by the OpenSSL Project\n * for use in the OpenSSL Toolkit. (http://www.openssl.org/)\"\n *\n * 4. The names \"OpenSSL Toolkit\" and \"OpenSSL Project\" must not be used to\n * endorse or promote products derived from this software without\n * prior written permission. For written permission, please contact\n * openssl-core@openssl.org.\n *\n * 5. Products derived from this software may not be called \"OpenSSL\"\n * nor may \"OpenSSL\" appear in their names without prior written\n * permission of the OpenSSL Project.\n *\n * 6. Redistributions of any form whatsoever must retain the following\n * acknowledgment:\n * \"This product includes software developed by the OpenSSL Project\n * for use in the OpenSSL Toolkit (http://www.openssl.org/)\"\n *\n * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY\n * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR\n * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,\n * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\n * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED\n * OF THE POSSIBILITY OF SUCH DAMAGE.\n * ====================================================================\n *\n * This product includes cryptographic software written by Eric Young\n * (eay@cryptsoft.com). This product includes software written by Tim\n * Hudson (tjh@cryptsoft.com).\n *\n */\n\n Original SSLeay License\n -----------------------\n\n/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)\n * All rights reserved.\n *\n * This package is an SSL implementation written\n * by Eric Young (eay@cryptsoft.com).\n * The implementation was written so as to conform with Netscapes SSL.\n * \n * This library is free for commercial and non-commercial use as long as\n * the following conditions are aheared to. The following conditions\n * apply to all code found in this distribution, be it the RC4, RSA,\n * lhash, DES, etc., code; not just the SSL code. The SSL documentation\n * included with this distribution is covered by the same copyright terms\n * except that the holder is Tim Hudson (tjh@cryptsoft.com).\n * \n * Copyright remains Eric Young's, and as such any Copyright notices in\n * the code are not to be removed.\n * If this package is used in a product, Eric Young should be given attribution\n * as the author of the parts of the library used.\n * This can be in the form of a textual message at program startup or\n * in documentation (online or textual) provided with the package.\n * \n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. All advertising materials mentioning features or use of this software\n * must display the following acknowledgement:\n * \"This product includes cryptographic software written by\n * Eric Young (eay@cryptsoft.com)\"\n * The word 'cryptographic' can be left out if the rouines from the library\n * being used are not cryptographic related :-).\n * 4. If you include any Windows specific code (or a derivative thereof) from \n * the apps directory (application code) you must include an acknowledgement:\n * \"This product includes software written by Tim Hudson (tjh@cryptsoft.com)\"\n * \n * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND\n * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n * SUCH DAMAGE.\n * \n * The licence and distribution terms for any publically available version or\n * derivative of this code cannot be changed. i.e. this code cannot simply be\n * copied and put under another distribution licence\n * [including the GNU Public Licence.]\n */\n\n\nISC license used for completely new code in BoringSSL:\n\n/* Copyright (c) 2015, Google Inc.\n *\n * Permission to use, copy, modify, and/or distribute this software for any\n * purpose with or without fee is hereby granted, provided that the above\n * copyright notice and this permission notice appear in all copies.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY\n * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION\n * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN\n * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */\n\n\nThe code in third_party/fiat carries the MIT license:\n\nCopyright (c) 2015-2016 the fiat-crypto authors (see\nhttps://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS).\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" - package_name: ring package_version: 0.17.8 repository: https://github.com/briansmith/ring - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/ring-0.17.8/LICENSE) + license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/ring-0.17.8/LICENSE) licenses: - - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/ring-0.17.8/LICENSE) + - license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/ring-0.17.8/LICENSE) text: "Note that it is easy for this file to get out of sync with the licenses in the\nsource code files. It's recommended to compare the licenses in the source code\nwith what's mentioned here.\n\n*ring* is derived from BoringSSL, so the licensing situation in *ring* is\nsimilar to BoringSSL.\n\n*ring* uses an ISC-style license like BoringSSL for code in new files,\nincluding in particular all the Rust code:\n\n Copyright 2015-2016 Brian Smith.\n\n Permission to use, copy, modify, and/or distribute this software for any\n purpose with or without fee is hereby granted, provided that the above\n copyright notice and this permission notice appear in all copies.\n\n THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHORS DISCLAIM ALL WARRANTIES\n WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY\n SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION\n OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN\n CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\n\nBoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL\nlicensing. Files that are completely new have a Google copyright and an ISC\nlicense. This license is reproduced at the bottom of this file.\n\nContributors to BoringSSL are required to follow the CLA rules for Chromium:\nhttps://cla.developers.google.com/clas\n\nFiles in third_party/ have their own licenses, as described therein. The MIT\nlicense, for third_party/fiat, which, unlike other third_party directories, is\ncompiled into non-test libraries, is included below.\n\nThe OpenSSL toolkit stays under a dual license, i.e. both the conditions of the\nOpenSSL License and the original SSLeay license apply to the toolkit. See below\nfor the actual license texts. Actually both licenses are BSD-style Open Source\nlicenses. In case of any license issues related to OpenSSL please contact\nopenssl-core@openssl.org.\n\nThe following are Google-internal bug numbers where explicit permission from\nsome authors is recorded for use of their work:\n 27287199\n 27287880\n 27287883\n\n OpenSSL License\n ---------------\n\n/* ====================================================================\n * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved.\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n *\n * 1. Redistributions of source code must retain the above copyright\n * notice, this list of conditions and the following disclaimer. \n *\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in\n * the documentation and/or other materials provided with the\n * distribution.\n *\n * 3. All advertising materials mentioning features or use of this\n * software must display the following acknowledgment:\n * \"This product includes software developed by the OpenSSL Project\n * for use in the OpenSSL Toolkit. (http://www.openssl.org/)\"\n *\n * 4. The names \"OpenSSL Toolkit\" and \"OpenSSL Project\" must not be used to\n * endorse or promote products derived from this software without\n * prior written permission. For written permission, please contact\n * openssl-core@openssl.org.\n *\n * 5. Products derived from this software may not be called \"OpenSSL\"\n * nor may \"OpenSSL\" appear in their names without prior written\n * permission of the OpenSSL Project.\n *\n * 6. Redistributions of any form whatsoever must retain the following\n * acknowledgment:\n * \"This product includes software developed by the OpenSSL Project\n * for use in the OpenSSL Toolkit (http://www.openssl.org/)\"\n *\n * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY\n * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR\n * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR\n * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT\n * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,\n * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\n * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED\n * OF THE POSSIBILITY OF SUCH DAMAGE.\n * ====================================================================\n *\n * This product includes cryptographic software written by Eric Young\n * (eay@cryptsoft.com). This product includes software written by Tim\n * Hudson (tjh@cryptsoft.com).\n *\n */\n\n Original SSLeay License\n -----------------------\n\n/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)\n * All rights reserved.\n *\n * This package is an SSL implementation written\n * by Eric Young (eay@cryptsoft.com).\n * The implementation was written so as to conform with Netscapes SSL.\n * \n * This library is free for commercial and non-commercial use as long as\n * the following conditions are aheared to. The following conditions\n * apply to all code found in this distribution, be it the RC4, RSA,\n * lhash, DES, etc., code; not just the SSL code. The SSL documentation\n * included with this distribution is covered by the same copyright terms\n * except that the holder is Tim Hudson (tjh@cryptsoft.com).\n * \n * Copyright remains Eric Young's, and as such any Copyright notices in\n * the code are not to be removed.\n * If this package is used in a product, Eric Young should be given attribution\n * as the author of the parts of the library used.\n * This can be in the form of a textual message at program startup or\n * in documentation (online or textual) provided with the package.\n * \n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions\n * are met:\n * 1. Redistributions of source code must retain the copyright\n * notice, this list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright\n * notice, this list of conditions and the following disclaimer in the\n * documentation and/or other materials provided with the distribution.\n * 3. All advertising materials mentioning features or use of this software\n * must display the following acknowledgement:\n * \"This product includes cryptographic software written by\n * Eric Young (eay@cryptsoft.com)\"\n * The word 'cryptographic' can be left out if the rouines from the library\n * being used are not cryptographic related :-).\n * 4. If you include any Windows specific code (or a derivative thereof) from \n * the apps directory (application code) you must include an acknowledgement:\n * \"This product includes software written by Tim Hudson (tjh@cryptsoft.com)\"\n * \n * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND\n * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\n * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\n * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE\n * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\n * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\n * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\n * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\n * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\n * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\n * SUCH DAMAGE.\n * \n * The licence and distribution terms for any publically available version or\n * derivative of this code cannot be changed. i.e. this code cannot simply be\n * copied and put under another distribution licence\n * [including the GNU Public Licence.]\n */\n\n\nISC license used for completely new code in BoringSSL:\n\n/* Copyright (c) 2015, Google Inc.\n *\n * Permission to use, copy, modify, and/or distribute this software for any\n * purpose with or without fee is hereby granted, provided that the above\n * copyright notice and this permission notice appear in all copies.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\" AND THE AUTHOR DISCLAIMS ALL WARRANTIES\n * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF\n * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY\n * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES\n * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION\n * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN\n * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */\n\n\nThe code in third_party/fiat carries the MIT license:\n\nCopyright (c) 2015-2016 the fiat-crypto authors (see\nhttps://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS).\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" - package_name: rmp - package_version: 0.8.12 + package_version: 0.8.14 repository: https://github.com/3Hren/msgpack-rust license: MIT licenses: @@ -17080,6 +17110,34 @@ third_party_libraries: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +- package_name: rmpv + package_version: 1.3.0 + repository: https://github.com/3Hren/msgpack-rust + license: MIT + licenses: + - license: MIT + text: | + MIT License + + Copyright (c) 2017 Evgeny Safronov + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE @@ -24836,9 +24894,9 @@ third_party_libraries: - package_name: webpki package_version: 0.22.4 repository: https://github.com/briansmith/webpki - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/webpki-0.22.4/LICENSE) + license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/webpki-0.22.4/LICENSE) licenses: - - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-6f17d22bba15001f/webpki-0.22.4/LICENSE) + - license: License specified in file ($CARGO_HOME/registry/src/github.com-1ecc6299db9ec823/webpki-0.22.4/LICENSE) text: | Except as otherwise noted, this project is licensed under the following (ISC-style) terms: diff --git a/benchmark/run_benchmarks_ci.sh b/benchmark/run_benchmarks_ci.sh new file mode 100755 index 000000000..57466d03d --- /dev/null +++ b/benchmark/run_benchmarks_ci.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +function message() { + local message=$1 verbose=${2:-"true"} + if [[ "${verbose}" == "true" ]]; then + echo "$(date +"%T%:z"): $message" + fi +} + +CURRENT_PATH=$(pwd) +readonly CURRENT_PATH="${CURRENT_PATH%/}" +readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" &>/dev/null && pwd 2>/dev/null)" +readonly PROJECT_DIR="${SCRIPT_DIR}/.." +OUTPUT_DIR="${1:-}" + +pushd "${PROJECT_DIR}" > /dev/null + +# Run benchmarks +# TODO https://datadoghq.atlassian.net/browse/APMSP-1228 +# Right now we are only running a single benchmark to test the CI setup, and there is only +# support for the Criterion SamplingMode::Flat in the new benchmarking framework. This will be +# worked on going forward. +message "Running benchmarks" +cargo bench -p datadog-trace-obfuscation --bench trace_obfuscation -- sql/obfuscate_sql_string +message "Finished running benchmarks" + +# Copy the benchmark results to the output directory +if [[ -n "${OUTPUT_DIR}" && -d "target" ]]; then + # Is this a relative path? + if [[ "$OUTPUT_DIR" != /* ]]; then + OUTPUT_DIR="${CURRENT_PATH}/${OUTPUT_DIR}" + fi + message "Copying benchmark results to ${OUTPUT_DIR}" + pushd target > /dev/null + find criterion -type d -regex '.*/new$' | while read -r dir; do + mkdir -p "${OUTPUT_DIR}/${dir}" + cp -r "${dir}"/* "${OUTPUT_DIR}/${dir}" + done + popd > /dev/null +fi diff --git a/ddsketch/Cargo.toml b/ddsketch/Cargo.toml new file mode 100644 index 000000000..50f84a4e4 --- /dev/null +++ b/ddsketch/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "datadog-ddsketch" +description = "Minimal implementation of Datadog's DDSketch" +edition.workspace = true +version.workspace = true +rust-version.workspace = true +license.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +prost = "0.11.6" + +[build-dependencies] +prost-build = { version = "0.11.9", optional = true } +protoc-bin-vendored = { version = "3.0.0", optional = true } + +[features] +generate-protobuf = ["dep:prost-build", "dep:protoc-bin-vendored"] diff --git a/ddsketch/build.rs b/ddsketch/build.rs new file mode 100644 index 000000000..189e71071 --- /dev/null +++ b/ddsketch/build.rs @@ -0,0 +1,52 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +use std::error::Error; + +#[cfg(feature = "generate-protobuf")] +use std::{ + env, + fs::File, + io::{Read, Write}, + path::Path, +}; + +#[cfg(feature = "generate-protobuf")] +const HEADER: &str = "// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +"; + +fn main() -> Result<(), Box> { + #[cfg(feature = "generate-protobuf")] + { + // protoc is required to compile proto files. This uses protobuf_src to compile protoc + // from the source, setting the env var to tell prost_build where to find it. + std::env::set_var("PROTOC", protoc_bin_vendored::protoc_bin_path().unwrap()); + + let mut config = prost_build::Config::new(); + + let cur_working_dir = env::var("CARGO_MANIFEST_DIR")?; + let output_path = Path::new(&cur_working_dir).join("src"); + + config.out_dir(output_path.clone()); + + println!("cargo:rerun-if-changed=src/pb/DDSketch.proto"); + config.compile_protos(&["src/pb/DDSketch.proto"], &["src/"])?; + + prepend_to_file(HEADER.as_bytes(), &output_path.join("pb.rs"))?; + } + + Ok(()) +} + +#[cfg(feature = "generate-protobuf")] +fn prepend_to_file(data: &[u8], file_path: &Path) -> Result<(), Box> { + let mut f = File::open(file_path)?; + let mut content = data.to_owned(); + f.read_to_end(&mut content)?; + + let mut f = File::create(file_path)?; + f.write_all(content.as_slice())?; + Ok(()) +} diff --git a/ddsketch/src/lib.rs b/ddsketch/src/lib.rs new file mode 100644 index 000000000..b23d05f07 --- /dev/null +++ b/ddsketch/src/lib.rs @@ -0,0 +1,464 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +//! This crate defines a minimal implementation of DDSketch. +//! +//! DDSketch is a data sketch used to generate percentiles over streaming data using constant +//! memory. A DDSketch is essentially a histogram that partitions the range of positive values into +//! an infinite number of indexed bins whose size grows exponentially. It keeps track of the number +//! of values (or possibly floating-point weights) added to each bin. Negative values are +//! partitioned like positive values, symmetrically to zero. The value zero as well as its close +//! neighborhood that would be mapped to extreme bin indexes is mapped to a specific counter. +use std::collections::{HashMap, VecDeque}; + +use prost::Message; + +/// Protobuf representation of DDSketch - generated by build.rs +#[rustfmt::skip] +pub mod pb; + +/// This is a minimal DDSketch implementation +/// +/// This implementation only supports a part of the standard (which is also only the parts dd +/// backend supports :shrug:) +/// - max length contiguous bin store, with lower bin +/// collapse behavior. +/// - Positive or zero values +/// +/// The default sketch has a 1% relative accuracy, and only accepts positive points +/// +/// See for the reference implementation +#[derive(Debug, Default, Clone)] +pub struct DDSketch { + store: LowCollapsingDenseStore, // Store the weight of each bin + zero_count: f64, // Store the weight of the bin of value 0 + mapping: LogMapping, // Bin-Value mapping +} + +impl DDSketch { + /// Return an iterator over `(value, weight)` pair for each bin + pub fn ordered_bins(&self) -> Vec<(f64, f64)> { + let mut bins: Vec<_> = std::iter::once((0.0, self.zero_count)) + .chain(self.store.bins().map(|(b, v)| (self.mapping.value(b), v))) + .collect(); + bins.sort_by(|a, b| a.0.total_cmp(&b.0)); + bins + } + + /// Add a point with value `point` to the sketch + /// `point` must be positive + pub fn add(&mut self, point: f64) -> Result<(), Box> { + self.add_with_count(point, 1.0) + } + + /// Add `count` point with value `point` to the sketch + /// `count` and `point` must be positive + pub fn add_with_count( + &mut self, + point: f64, + count: f64, + ) -> Result<(), Box> { + if count.is_nan() || count.is_infinite() { + return Err("count is invalid".into()); + } + if point < 0.0 || point.is_nan() || point.is_infinite() { + return Err("point is invalid".into()); + } else if point < self.mapping.min_indexable_value { + self.zero_count += count; + } else { + let index = self.mapping.index(point); + *self.store.bin_mut(index) += count; + } + Ok(()) + } + + /// Return a protobuf of the sketch + pub fn into_pb(self) -> pb::DdSketch { + let contiguous_bins: Vec = self.store.bins.into(); + pb::DdSketch { + mapping: Some(pb::IndexMapping { + gamma: self.mapping.gamma, + index_offset: self.mapping.index_offset, + interpolation: pb::index_mapping::Interpolation::None.into(), + }), + positive_values: Some(pb::Store { + bin_counts: HashMap::new(), + contiguous_bin_counts: contiguous_bins, + contiguous_bin_index_offset: self.store.offset, + }), + zero_count: self.zero_count, + negative_values: Some(pb::Store { + bin_counts: HashMap::new(), + contiguous_bin_counts: Vec::new(), + contiguous_bin_index_offset: 0, + }), + } + } + + /// Return a serialized protobuf of the sketch + pub fn encode_to_vec(self) -> Vec { + self.into_pb().encode_to_vec() + } +} + +/// A store mapping the bin indexes to their respective weights +/// +/// Stores the weights as contiguousBinCounts, only the bins within `offset` and the highest +/// non empty bin are stored. +/// +/// Stores the weights of a contiguous range of bins containing all non-empty bins. The range start +/// at index `offset` and end at index `offset + bins.len()` +/// +/// The range of stored bins is updated when accessing the index of a bin out of the range +/// with [`Self::bin_idx_to_bin_idx()`]. If the `max_size` is reached the lower bins are +/// collapsed together to free space. +#[derive(Debug, Clone)] +struct LowCollapsingDenseStore { + bins: VecDeque, + offset: i32, + max_size: i32, +} + +impl LowCollapsingDenseStore { + fn new(max_size: i32) -> Option { + if max_size < 0 { + return None; + } + Some(Self { + bins: VecDeque::new(), + offset: 0, + max_size, + }) + } + + /// Return an iterator over the bins + /// The iterator yields pairs `(bin_index, count)` + fn bins(&self) -> impl Iterator + '_ { + self.bins + .iter() + .enumerate() + .map(|(i, &v)| (i as i32 + self.offset, v)) + } + + /// Return a mutable reference to the bin at index `bin_index` + fn bin_mut(&mut self, bin_index: i32) -> &mut f64 { + let store_index = self.bin_idx_to_store_idx(bin_index); + &mut self.bins[store_index] + } + + /// Return the `store_index` of the bin at index `bin_index` + fn bin_idx_to_store_idx(&mut self, bin_index: i32) -> usize { + if self.bins.is_empty() { + // If the bins are empty, start them at the index + self.offset = bin_index; + self.bins.push_back(0.0); + return 0; + } + + // General case + // Bucket lower than the stored range + if bin_index < self.offset { + let additional_low_bins = self.offset - bin_index; + debug_assert!(additional_low_bins >= 0); + + let additional_low_bins = std::cmp::min( + additional_low_bins as usize, + self.max_size as usize - self.bins.len(), + ); + + self.bins.reserve(additional_low_bins); + for _ in 0..additional_low_bins { + self.bins.push_front(0.0); + } + + self.offset -= additional_low_bins as i32; + 0 + } + // Bucket higher than the stored range + else if self.offset + self.bins.len() as i32 <= bin_index { + let bin_range_size = bin_index - self.offset + 1; // Number of bucket to store + + if bin_range_size > self.max_size { + self.collapse_low_bins(bin_range_size - self.max_size); + } + debug_assert!(self.bins.len() as i32 <= self.max_size); + + let store_index = bin_index - self.offset; + for _ in 0..(store_index - self.bins.len() as i32 + 1) { + self.bins.push_back(0.0); + } + store_index as usize + } + // Bucket within the stored range + else { + (bin_index - self.offset) as usize + } + } + + /// Collapse the `bin_number` lowest bins + fn collapse_low_bins(&mut self, bin_number: i32) { + let mut count = 0.0; + for _ in 0..bin_number { + count += self.bins.pop_front().unwrap_or(0.0); + } + if let Some(lowest_bin) = self.bins.front_mut() { + *lowest_bin += count; + } else { + self.bins.push_front(count); + } + self.offset += bin_number; + } +} + +impl Default for LowCollapsingDenseStore { + fn default() -> Self { + Self::new(2048).unwrap() + } +} + +/// Logarithmic mapping of bucket index to value +#[derive(Debug, Clone, Copy)] +struct LogMapping { + gamma: f64, + multiplier: f64, + min_indexable_value: f64, + index_offset: f64, +} + +impl LogMapping { + fn new(gamma: f64, offset: f64) -> Option { + if gamma <= 1.0 { + return None; + } + let multiplier = Self::multiplier_from_gamma(gamma); + Some(Self { + gamma, + multiplier, + min_indexable_value: max( + // So that the value representing the lowest bucket is >= std::f64::MIN_POSITIVE + f64::MIN_POSITIVE * gamma, + // Minimum value so that index >= i32::MIN + ((i32::MIN as f64 - offset) / multiplier + 1.0).exp(), + )?, + index_offset: offset, + }) + } + + /// Returns the multiplier used to convert ln to base-gamma logarithm + fn multiplier_from_gamma(gamma: f64) -> f64 { + 1.0 / gamma.ln() + } + + /// Returns the relative accuracy guaranteed by the mapping + fn relative_accuracy(&self) -> f64 { + 1.0 - 2.0 / (1.0 + self.gamma) + } + + /// Returns the index of the bucket containing `value` + fn index(&self, value: f64) -> i32 { + (value.ln() * self.multiplier + self.index_offset).floor() as i32 + } + + /// Returns the value representing the bucket at `index` + fn value(&self, index: i32) -> f64 { + ((index as f64 - self.index_offset) / self.multiplier).exp() + * (1.0 + self.relative_accuracy()) + } +} + +impl Default for LogMapping { + fn default() -> Self { + const RELATIVE_ACCURACY: f64 = 0.007751937984496124; + const GAMMA: f64 = (1.0 + RELATIVE_ACCURACY) / (1.0 - RELATIVE_ACCURACY); + + const BACKEND_SKETCH_MIN_VALUE: f64 = 1e-9; + // offset used in datadog's backend for sketches + let offset: f64 = (1.0 - (BACKEND_SKETCH_MIN_VALUE.ln() / GAMMA.ln()).floor()) + 0.5; + + Self::new(GAMMA, offset).unwrap() + } +} + +fn max(a: f64, b: f64) -> Option { + if a.is_nan() || b.is_nan() { + None + } else if a > b { + Some(a) + } else { + Some(b) + } +} + +#[cfg(test)] +mod test { + use prost::Message; + + use super::*; + + macro_rules! assert_within { + ($x:expr, $y:expr, $tolerance:expr) => { + let diff = $x - $y; + assert!( + -$tolerance < diff && diff < $tolerance, + "x: {} y: {}", + $x, + $y, + ); + }; + } + + #[test] + fn test_exponential_mapping_within_tolerances() { + let mapping = LogMapping::default(); + + let values: &[f64] = &[1e-30, 0.1, 2.0, 10.0, 25.0, 10000.0]; + for &value in values { + let index = mapping.index(value); + let value_bucket = mapping.value(index); + + assert_within!(value_bucket / value, 1.0, 0.01); + } + } + + #[test] + fn test_exponential_mapping_relative_accuracy() { + let mapping = LogMapping::default(); + + assert_within!( + mapping.relative_accuracy(), + 0.007751937984496138, + f64::EPSILON + ); + } + + #[test] + fn test_sketch_add() { + let mut sketch = DDSketch::default(); + let points: &[f64] = &[0.0, 1e-5, 0.1, 2.0, 10.0, 25.0, 10000.0]; + for (i, &point) in points.iter().enumerate() { + assert!(sketch.add_with_count(point, i as f64 + 1.0).is_ok()); + } + + dbg!(sketch.store.bins.len(), sketch.store.offset); + + for (i, (value, count)) in sketch + .ordered_bins() + .into_iter() + .filter(|(_, p)| *p != 0.0) + .enumerate() + { + if points[i] == 0.0 { + assert_within!(value, 0.0, f64::EPSILON); + assert_within!(count, i as f64 + 1.0, f64::EPSILON); + } else { + assert_within!(value / points[i], 1.0, 0.01); + assert_within!(count, i as f64 + 1.0, f64::EPSILON); + } + } + } + + #[test] + fn test_skecth_add_negative() { + let mut sketch = DDSketch::default(); + assert!(sketch.add(-1.0).is_err()); + } + + #[test] + fn test_skecth_add_nan() { + let mut sketch = DDSketch::default(); + assert!(sketch.add(f64::NAN).is_err()); + } + + #[test] + fn test_skecth_encode() { + let mut sketch = DDSketch::default(); + let points: &[f64] = &[0.0, 1e-30, 0.1, 2.0, 10.0, 25.0, 10000.0]; + for (i, &point) in points.iter().enumerate() { + assert!(sketch.add_with_count(point, i as f64).is_ok()); + } + + let pb_sketch = sketch.into_pb().encode_to_vec(); + assert!(!pb_sketch.is_empty()); + } + + #[test] + fn test_low_collapsing_store() { + let mut store = LowCollapsingDenseStore::new(5).unwrap(); + + // Test initial push up to capacity + for i in 0..5 { + *store.bin_mut(i + 10) = 1.0; + } + for (i, b) in store.bins().enumerate() { + assert_eq!(b.0, i as i32 + 10); + assert_eq!(b.1, 1.0) + } + + // Indexing existing bins + for i in 0..5 { + *store.bin_mut(i + 10) += 1.0; + } + for (i, b) in store.bins().enumerate() { + assert_eq!(b.0, i as i32 + 10); + assert_eq!(b.1, 2.0) + } + } + + #[test] + fn test_low_collapsing_store_low_bins_are_collapsed() { + let mut store = LowCollapsingDenseStore::new(5).unwrap(); + + // Test initial push up to capacity to max + for i in 0..5 { + *store.bin_mut(i + 10) = 1.0; + } + + // Indexing low bins at max capacity + for i in 0..3 { + *store.bin_mut(i) += 1.0; + } + for (i, b) in store.bins().enumerate() { + assert_eq!(b.0, i as i32 + 10); + if i == 0 { + assert_eq!(b.1, 4.0) + } else { + assert_eq!(b.1, 1.0) + } + } + + // Indexing higer bins collapses lower bins + *store.bin_mut(15) = 1.0; + for (i, b) in store.bins().enumerate() { + assert_eq!(b.0, i as i32 + 11); + if i == 0 { + assert_eq!(b.1, 5.0) + } else { + assert_eq!(b.1, 1.0) + } + } + } + + #[test] + fn test_low_collapsing_store_up_expansion() { + let mut store = LowCollapsingDenseStore::new(3).unwrap(); + + *store.bin_mut(1) = 1.0; + *store.bin_mut(3) = 1.0; + assert_eq!( + store.bins().collect::>(), + &[(1, 1.0), (2, 0.0), (3, 1.0)] + ) + } + + #[test] + fn test_low_collapsing_store_down_expansion() { + let mut store = LowCollapsingDenseStore::new(3).unwrap(); + + *store.bin_mut(3) = 1.0; + *store.bin_mut(1) = 1.0; + assert_eq!( + store.bins().collect::>(), + &[(1, 1.0), (2, 0.0), (3, 1.0)] + ) + } +} diff --git a/ddsketch/src/pb.rs b/ddsketch/src/pb.rs new file mode 100644 index 000000000..da8f73bc7 --- /dev/null +++ b/ddsketch/src/pb.rs @@ -0,0 +1,107 @@ +// Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ +// SPDX-License-Identifier: Apache-2.0 + +/// A DDSketch is essentially a histogram that partitions the range of positive values into an infinite number of +/// indexed bins whose size grows exponentially. It keeps track of the number of values (or possibly floating-point +/// weights) added to each bin. Negative values are partitioned like positive values, symmetrically to zero. +/// The value zero as well as its close neighborhood that would be mapped to extreme bin indexes is mapped to a specific +/// counter. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DdSketch { + /// The mapping between positive values and the bin indexes they belong to. + #[prost(message, optional, tag = "1")] + pub mapping: ::core::option::Option, + /// The store for keeping track of positive values. + #[prost(message, optional, tag = "2")] + pub positive_values: ::core::option::Option, + /// The store for keeping track of negative values. A negative value v is mapped using its positive opposite -v. + #[prost(message, optional, tag = "3")] + pub negative_values: ::core::option::Option, + /// The count for the value zero and its close neighborhood (whose width depends on the mapping). + #[prost(double, tag = "4")] + pub zero_count: f64, +} +/// How to map positive values to the bins they belong to. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct IndexMapping { + /// The gamma parameter of the mapping, such that bin index that a value v belongs to is roughly equal to + /// log(v)/log(gamma). + #[prost(double, tag = "1")] + pub gamma: f64, + /// An offset that can be used to shift all bin indexes. + #[prost(double, tag = "2")] + pub index_offset: f64, + /// To speed up the computation of the index a value belongs to, the computation of the log may be approximated using + /// the fact that the log to the base 2 of powers of 2 can be computed at a low cost from the binary representation of + /// the input value. Other values can be approximated by interpolating between successive powers of 2 (linearly, + /// quadratically or cubically). + /// NONE means that the log is to be computed exactly (no interpolation). + #[prost(enumeration = "index_mapping::Interpolation", tag = "3")] + pub interpolation: i32, +} +/// Nested message and enum types in `IndexMapping`. +pub mod index_mapping { + #[derive( + Clone, + Copy, + Debug, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + ::prost::Enumeration + )] + #[repr(i32)] + pub enum Interpolation { + None = 0, + Linear = 1, + Quadratic = 2, + Cubic = 3, + } + impl Interpolation { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Interpolation::None => "NONE", + Interpolation::Linear => "LINEAR", + Interpolation::Quadratic => "QUADRATIC", + Interpolation::Cubic => "CUBIC", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "NONE" => Some(Self::None), + "LINEAR" => Some(Self::Linear), + "QUADRATIC" => Some(Self::Quadratic), + "CUBIC" => Some(Self::Cubic), + _ => None, + } + } + } +} +/// A Store maps bin indexes to their respective counts. +/// Counts can be encoded sparsely using binCounts, but also in a contiguous way using contiguousBinCounts and +/// contiguousBinIndexOffset. Given that non-empty bins are in practice usually contiguous or close to one another, the +/// latter contiguous encoding method is usually more efficient than the sparse one. +/// Both encoding methods can be used conjointly. If a bin appears in both the sparse and the contiguous encodings, its +/// count value is the sum of the counts in each encodings. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Store { + /// The bin counts, encoded sparsely. + #[prost(map = "sint32, double", tag = "1")] + pub bin_counts: ::std::collections::HashMap, + /// The bin counts, encoded contiguously. The values of contiguousBinCounts are the counts for the bins of indexes + /// o, o+1, o+2, etc., where o is contiguousBinIndexOffset. + #[prost(double, repeated, tag = "2")] + pub contiguous_bin_counts: ::prost::alloc::vec::Vec, + #[prost(sint32, tag = "3")] + pub contiguous_bin_index_offset: i32, +} diff --git a/ddsketch/src/pb/DDSketch.proto b/ddsketch/src/pb/DDSketch.proto new file mode 100644 index 000000000..a76632485 --- /dev/null +++ b/ddsketch/src/pb/DDSketch.proto @@ -0,0 +1,65 @@ +/* Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/ + * SPDX-License-Identifier: Apache-2.0 + */ + +syntax = "proto3"; + +package pb; + +// A DDSketch is essentially a histogram that partitions the range of positive values into an infinite number of +// indexed bins whose size grows exponentially. It keeps track of the number of values (or possibly floating-point +// weights) added to each bin. Negative values are partitioned like positive values, symmetrically to zero. +// The value zero as well as its close neighborhood that would be mapped to extreme bin indexes is mapped to a specific +// counter. +message DDSketch { + // The mapping between positive values and the bin indexes they belong to. + IndexMapping mapping = 1; + + // The store for keeping track of positive values. + Store positiveValues = 2; + + // The store for keeping track of negative values. A negative value v is mapped using its positive opposite -v. + Store negativeValues = 3; + + // The count for the value zero and its close neighborhood (whose width depends on the mapping). + double zeroCount = 4; +} + +// How to map positive values to the bins they belong to. +message IndexMapping { + // The gamma parameter of the mapping, such that bin index that a value v belongs to is roughly equal to + // log(v)/log(gamma). + double gamma = 1; + + // An offset that can be used to shift all bin indexes. + double indexOffset = 2; + + // To speed up the computation of the index a value belongs to, the computation of the log may be approximated using + // the fact that the log to the base 2 of powers of 2 can be computed at a low cost from the binary representation of + // the input value. Other values can be approximated by interpolating between successive powers of 2 (linearly, + // quadratically or cubically). + // NONE means that the log is to be computed exactly (no interpolation). + Interpolation interpolation = 3; + enum Interpolation { + NONE = 0; + LINEAR = 1; + QUADRATIC = 2; + CUBIC = 3; + } +} + +// A Store maps bin indexes to their respective counts. +// Counts can be encoded sparsely using binCounts, but also in a contiguous way using contiguousBinCounts and +// contiguousBinIndexOffset. Given that non-empty bins are in practice usually contiguous or close to one another, the +// latter contiguous encoding method is usually more efficient than the sparse one. +// Both encoding methods can be used conjointly. If a bin appears in both the sparse and the contiguous encodings, its +// count value is the sum of the counts in each encodings. +message Store { + // The bin counts, encoded sparsely. + map binCounts = 1; + + // The bin counts, encoded contiguously. The values of contiguousBinCounts are the counts for the bins of indexes + // o, o+1, o+2, etc., where o is contiguousBinIndexOffset. + repeated double contiguousBinCounts = 2 [packed = true]; + sint32 contiguousBinIndexOffset = 3; +} \ No newline at end of file diff --git a/ddtelemetry/Cargo.toml b/ddtelemetry/Cargo.toml index 68f04ae51..7aab69a9f 100644 --- a/ddtelemetry/Cargo.toml +++ b/ddtelemetry/Cargo.toml @@ -13,6 +13,8 @@ tracing = ["tracing/std"] [dependencies] anyhow = { version = "1.0" } ddcommon = { path = "../ddcommon" } +datadog-ddsketch = { path = "../ddsketch" } +base64 = "0.22" futures = { version = "0.3", default-features = false } http = "0.2" hyper = { version = "0.14", features = ["client"], default-features = false } diff --git a/ddtelemetry/examples/tm-send-sketch.rs b/ddtelemetry/examples/tm-send-sketch.rs new file mode 100644 index 000000000..0901e006b --- /dev/null +++ b/ddtelemetry/examples/tm-send-sketch.rs @@ -0,0 +1,121 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. This product includes software +// developed at Datadog (https://www.datadoghq.com/). Copyright 2023-Present +// Datadog, Inc. + +use std::{ + borrow::Cow, + sync::atomic::{AtomicU64, Ordering}, + time::SystemTime, +}; + +use ddcommon::Endpoint; +use ddtelemetry::{ + build_host, + config::Config, + data::{self, metrics::Distribution, Application, Telemetry}, + worker::http_client::request_builder, +}; +use http::{header::CONTENT_TYPE, Uri}; + +fn seq_id() -> u64 { + static SEQ_ID: AtomicU64 = AtomicU64::new(1); + SEQ_ID.fetch_add(1, Ordering::SeqCst) +} + +fn build_request<'a>( + application: &'a data::Application, + host: &'a data::Host, + payload: &'a data::Payload, +) -> data::Telemetry<'a> { + data::Telemetry { + api_version: data::ApiVersion::V1, + tracer_time: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + runtime_id: "runtime_id", + seq_id: seq_id(), + application, + host, + payload, + } +} + +pub async fn push_telemetry(config: &Config, telemetry: &Telemetry<'_>) -> anyhow::Result<()> { + let client = ddtelemetry::worker::http_client::from_config(config); + let req = request_builder(config)? + .method(http::Method::POST) + .header(CONTENT_TYPE, ddcommon::header::APPLICATION_JSON) + .header("dd-telemetry-debug-enabled", "true") + .body(serde_json::to_string(telemetry)?.into())?; + + let resp = client.request(req).await?; + + if !resp.status().is_success() { + Err(anyhow::Error::msg(format!( + "Telemetry error: response status: {}", + resp.status() + ))) + } else { + Ok(()) + } +} + +fn main() { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async_main()) +} + +async fn async_main() { + let app = Application { + service_name: "sketch_test".to_owned(), + service_version: None, + env: None, + language_name: String::from("nodejs"), + language_version: String::from("0.0.0"), + tracer_version: String::from("n/a"), + runtime_name: None, + runtime_version: None, + runtime_patches: None, + }; + let host = build_host(); + + let mut sketch = datadog_ddsketch::DDSketch::default(); + for i in 0..1000 { + for j in 0..1000 { + sketch.add((i + j) as f64 / 1000.0).unwrap(); + } + } + + let payload = data::Payload::Sketches(data::Distributions { + series: vec![Distribution { + namespace: data::metrics::MetricNamespace::Telemetry, + tags: Vec::new(), + common: true, + metric: "telemetry_api.ms".to_owned(), + sketch: data::metrics::SerializedSketch::B64 { + sketch_b64: base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + sketch.encode_to_vec(), + ), + }, + _type: data::metrics::MetricType::Distribution, + interval: 10, + }], + }); + + let req = build_request(&app, &host, &payload); + + let mut config = Config::get().clone(); + config.endpoint = Some(Endpoint { + url: Uri::from_static( + "https://instrumentation-telemetry-intake.datad0g.com/api/v2/apmtelemetry", + ), + api_key: Some(Cow::Owned(std::env::var("DD_API_KEY").unwrap())), + }); + push_telemetry(&config, &req).await.unwrap(); +} diff --git a/ddtelemetry/src/data/metrics.rs b/ddtelemetry/src/data/metrics.rs index 50a129a43..b89186ae6 100644 --- a/ddtelemetry/src/data/metrics.rs +++ b/ddtelemetry/src/data/metrics.rs @@ -21,9 +21,19 @@ pub struct Distribution { pub namespace: MetricNamespace, pub metric: String, pub tags: Vec, - pub points: Vec, + #[serde(flatten)] + pub sketch: SerializedSketch, pub common: bool, pub interval: u64, + #[serde(rename = "type")] + pub _type: MetricType, +} + +#[derive(Serialize, Debug)] +#[serde(untagged)] +pub enum SerializedSketch { + Bytes { sketch: Vec }, + B64 { sketch_b64: String }, } #[derive(Serialize, Deserialize, Debug, Clone, Copy)] diff --git a/ddtelemetry/src/data/payload.rs b/ddtelemetry/src/data/payload.rs index 7fb9e6d65..d96600853 100644 --- a/ddtelemetry/src/data/payload.rs +++ b/ddtelemetry/src/data/payload.rs @@ -15,7 +15,7 @@ pub enum Payload { AppHeartbeat(#[serde(skip_serializing)] ()), AppClosing(#[serde(skip_serializing)] ()), GenerateMetrics(GenerateMetrics), - Distributions(Distributions), + Sketches(Distributions), Logs(Vec), MessageBatch(Vec), AppExtendedHeartbeat(AppStarted), @@ -32,7 +32,7 @@ impl Payload { AppHeartbeat(_) => "app-heartbeat", AppClosing(_) => "app-closing", GenerateMetrics(_) => "generate-metrics", - Distributions(_) => "distributions", + Sketches(_) => "sketches", Logs(_) => "logs", MessageBatch(_) => "message-batch", AppExtendedHeartbeat(_) => "app-extended-heartbeat", diff --git a/ddtelemetry/src/data/payloads.rs b/ddtelemetry/src/data/payloads.rs index 3dccc035f..9a521cd16 100644 --- a/ddtelemetry/src/data/payloads.rs +++ b/ddtelemetry/src/data/payloads.rs @@ -39,6 +39,7 @@ pub enum ConfigurationOrigin { #[derive(Serialize, Debug)] pub struct AppStarted { + #[serde(skip_serializing_if = "Vec::is_empty")] pub configuration: Vec, } diff --git a/ddtelemetry/src/metrics.rs b/ddtelemetry/src/metrics.rs index 5072ae301..c13f0512b 100644 --- a/ddtelemetry/src/metrics.rs +++ b/ddtelemetry/src/metrics.rs @@ -7,6 +7,7 @@ use std::{ time, }; +use datadog_ddsketch::DDSketch; use ddcommon::tag::Tag; use serde::{Deserialize, Serialize}; @@ -60,7 +61,7 @@ struct BucketKey { pub struct MetricBuckets { buckets: HashMap, series: HashMap>, - distributions: HashMap>, + distributions: HashMap, } #[derive(Default, Serialize, Deserialize)] @@ -101,7 +102,7 @@ impl MetricBuckets { pub fn flush_distributions( &mut self, - ) -> impl Iterator, Vec)> + '_ { + ) -> impl Iterator, DDSketch)> + '_ { self.distributions.drain().map( |( BucketKey { @@ -134,10 +135,7 @@ impl MetricBuckets { }) .add_point(point), metrics::MetricType::Distribution => { - self.distributions - .entry(bucket_key) - .or_default() - .push(point); + let _ = self.distributions.entry(bucket_key).or_default().add(point); } } } @@ -148,7 +146,16 @@ impl MetricBuckets { series: self.series.len() as u32, series_points: self.series.values().map(|v| v.len() as u32).sum(), distributions: self.distributions.len() as u32, - distributions_points: self.distributions.values().map(|v| v.len() as u32).sum(), + distributions_points: self + .distributions + .values() + .flat_map(|sketch| { + sketch + .ordered_bins() + .into_iter() + .map(|(_, weight)| weight as u32) + }) + .sum(), } } } @@ -226,6 +233,7 @@ mod tests { use super::*; use crate::data::metrics::{MetricNamespace, MetricType}; + /// Check if a and b are approximately equal with the given precision or 1.0e-6 by default macro_rules! assert_approx_eq { ($a:expr, $b:expr) => {{ let (a, b) = (&$a, &$b); @@ -236,6 +244,15 @@ mod tests { *b ); }}; + ($a:expr, $b:expr, $precision:expr) => {{ + let (a, b) = (&$a, &$b); + assert!( + (*a - *b).abs() < $precision, + "{} is not approximately equal to {}", + *a, + *b + ); + }}; } // Test util used to run assertions against an unsorted list @@ -243,7 +260,7 @@ mod tests { elems: T, assertions: &[&dyn Fn(&U) -> bool], ) { - let used = vec![false; assertions.len()]; + let mut used = vec![false; assertions.len()]; for e in elems { let mut found = false; for (i, &a) in assertions.iter().enumerate() { @@ -252,6 +269,7 @@ mod tests { panic!("Assertion {i} has been used multiple times"); } found = true; + used[i] = true; break; } } @@ -341,4 +359,169 @@ mod tests { ], ); } + + #[test] + fn test_distributions() { + let mut buckets = MetricBuckets::default(); + let contexts = MetricContexts::default(); + + let context_key_distribution = contexts.register_metric_context( + "metric_distribution".into(), + Vec::new(), + MetricType::Distribution, + false, + MetricNamespace::Tracers, + ); + let context_key_distribution_2 = contexts.register_metric_context( + "metric_distribution_2".into(), + Vec::new(), + MetricType::Distribution, + false, + MetricNamespace::Tracers, + ); + let extra_tags = vec![tag!("service", "foo")]; + + // Create 2 distributions with 2 and 3 points + buckets.add_point(context_key_distribution, 1.0, Vec::new()); + buckets.add_point(context_key_distribution, 1.0, Vec::new()); + buckets.add_point(context_key_distribution, 100.0, Vec::new()); + buckets.add_point(context_key_distribution, 1000.0, Vec::new()); + + buckets.add_point(context_key_distribution_2, 2.0, Vec::new()); + buckets.add_point(context_key_distribution_2, 200.0, Vec::new()); + + buckets.add_point(context_key_distribution_2, 3.0, extra_tags.clone()); + buckets.add_point(context_key_distribution_2, 300.0, extra_tags.clone()); + + let distributions: Vec<_> = buckets.flush_distributions().collect(); + + check_iter( + distributions.iter(), + &[ + &|(c, t, points)| { + if !(c == &context_key_distribution && t.is_empty()) { + return false; + } + let bins: Vec<_> = points + .ordered_bins() + .into_iter() + .filter(|(_, w)| *w != 0.0) + .collect(); + assert_eq!(bins.len(), 3); + // The precision is quite low since it is up to the ddsketch implementation to + // test the precision + assert_approx_eq!(bins[0].0, 1.0, 1.0e-1); + assert_approx_eq!(bins[0].1, 2.0); + assert_approx_eq!(bins[1].0, 100.0, 1.0); + assert_approx_eq!(bins[1].1, 1.0); + assert_approx_eq!(bins[2].0, 1000.0, 10.0); + assert_approx_eq!(bins[2].1, 1.0); + true + }, + &|(c, t, points)| { + if !(c == &context_key_distribution_2 && t.is_empty()) { + return false; + } + let bins: Vec<_> = points + .ordered_bins() + .into_iter() + .filter(|(_, w)| *w != 0.0) + .collect(); + assert_eq!(bins.len(), 2); + assert_approx_eq!(bins[0].0, 2.0, 1.0e-1); + assert_approx_eq!(bins[0].1, 1.0); + assert_approx_eq!(bins[1].0, 200.0, 1.0); + assert_approx_eq!(bins[1].1, 1.0); + true + }, + &|(c, t, points)| { + if !(c == &context_key_distribution_2 && !t.is_empty()) { + return false; + } + let bins: Vec<_> = points + .ordered_bins() + .into_iter() + .filter(|(_, w)| *w != 0.0) + .collect(); + assert_eq!(bins.len(), 2); + assert_approx_eq!(bins[0].0, 3.0, 1.0e-1); + assert_approx_eq!(bins[0].1, 1.0); + assert_approx_eq!(bins[1].0, 300.0, 1.0); + assert_approx_eq!(bins[1].1, 1.0); + true + }, + ], + ) + } + + #[test] + fn test_stats() { + let mut buckets = MetricBuckets::default(); + let contexts = MetricContexts::default(); + + let context_key_1 = contexts.register_metric_context( + "metric1".into(), + Vec::new(), + MetricType::Count, + false, + MetricNamespace::Tracers, + ); + + let context_key_2 = contexts.register_metric_context( + "metric2".into(), + Vec::new(), + MetricType::Gauge, + false, + MetricNamespace::Tracers, + ); + + let context_key_distribution = contexts.register_metric_context( + "metric_distribution".into(), + Vec::new(), + MetricType::Distribution, + false, + MetricNamespace::Tracers, + ); + + let context_key_distribution_2 = contexts.register_metric_context( + "metric_distribution_2".into(), + Vec::new(), + MetricType::Distribution, + false, + MetricNamespace::Tracers, + ); + + // Create 2 series with 2 and 3 points + buckets.add_point(context_key_1, 1.0, Vec::new()); + buckets.add_point(context_key_2, 2.0, Vec::new()); + buckets.flush_agregates(); + + buckets.add_point(context_key_1, 1.0, Vec::new()); + buckets.add_point(context_key_2, 2.0, Vec::new()); + buckets.flush_agregates(); + + buckets.add_point(context_key_1, 1.1, Vec::new()); + buckets.add_point(context_key_1, 2.1, Vec::new()); + buckets.flush_agregates(); + + // Create 2 buckets + buckets.add_point(context_key_1, 1.0, Vec::new()); + buckets.add_point(context_key_2, 2.0, Vec::new()); + + // Create 2 distributions with 2 and 3 points + buckets.add_point(context_key_distribution, 1.0, Vec::new()); + buckets.add_point(context_key_distribution, 1.1, Vec::new()); + buckets.add_point(context_key_distribution, 1.2, Vec::new()); + + buckets.add_point(context_key_distribution_2, 2.0, Vec::new()); + buckets.add_point(context_key_distribution_2, 2.1, Vec::new()); + + let stats = buckets.stats(); + + assert_eq!(stats.buckets, 2); + assert_eq!(stats.series, 2); + assert_eq!(stats.series_points, 5); + assert_eq!(stats.distributions, 2); + assert_eq!(stats.distributions_points, 5); + } } diff --git a/ddtelemetry/src/worker/mod.rs b/ddtelemetry/src/worker/mod.rs index 49d902b42..77a9cb024 100644 --- a/ddtelemetry/src/worker/mod.rs +++ b/ddtelemetry/src/worker/mod.rs @@ -511,7 +511,7 @@ impl TelemetryWorker { } let distributions = self.build_metrics_distributions(); if !distributions.series.is_empty() { - payloads.push(data::Payload::Distributions(distributions)) + payloads.push(data::Payload::Sketches(distributions)) } payloads } @@ -530,8 +530,14 @@ impl TelemetryWorker { namespace: context.namespace, metric: context.name.clone(), tags, - points, + sketch: data::metrics::SerializedSketch::B64 { + sketch_b64: base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + points.encode_to_vec(), + ), + }, common: context.common, + _type: context.metric_type, interval: MetricBuckets::METRICS_FLUSH_INTERVAL.as_secs(), }); } @@ -601,8 +607,7 @@ impl TelemetryWorker { } } AppHeartbeat(()) | AppClosing(()) => {} - // TODO Paul lgdc keep metrics until we know if the flush was a success - GenerateMetrics(_) | Distributions(_) => {} + GenerateMetrics(_) | Sketches(_) => {} } } diff --git a/rustfmt.toml b/rustfmt.toml index 44db20e30..479c60b69 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -6,7 +6,6 @@ ignore = [ ] format_macro_matchers = true - format_code_in_doc_comments=true wrap_comments = true max_width=100 diff --git a/sidecar/src/self_telemetry.rs b/sidecar/src/self_telemetry.rs index f2cbc539e..18a304211 100644 --- a/sidecar/src/self_telemetry.rs +++ b/sidecar/src/self_telemetry.rs @@ -28,8 +28,7 @@ struct MetricData<'a> { trace_api_requests: ContextKey, trace_api_responses: ContextKey, trace_api_errors: ContextKey, - // TODO: APMSP-1157 - Enable this metric when support is enabled. - // trace_api_bytes: ContextKey, + trace_api_bytes: ContextKey, trace_chunks_sent: ContextKey, trace_chunks_dropped: ContextKey, } @@ -107,14 +106,13 @@ impl<'a> MetricData<'a> { ], )); } - // TODO: APMSP-1157 - Enable this metric when support is enabled. - // if trace_metrics.bytes_sent > 0 { - // futures.push(self.send( - // self.trace_api_bytes, - // trace_metrics.bytes_sent as f64, - // vec![tag!("src_library", "libdatadog")], - // )); - // } + if trace_metrics.bytes_sent > 0 { + futures.push(self.send( + self.trace_api_bytes, + trace_metrics.bytes_sent as f64, + vec![tag!("src_library", "libdatadog")], + )); + } if trace_metrics.chunks_sent > 0 { futures.push(self.send( self.trace_chunks_sent, @@ -253,14 +251,13 @@ impl SelfTelemetry { true, MetricNamespace::Tracers, ), - // TODO: APMSP-1157 - Enable this metric when support is enabled. - // trace_api_bytes: worker.register_metric_context( - // "trace_api_bytes".to_string(), - // vec![], - // MetricType::Distribution, - // true, - // MetricNamespace::Tracers, - // ), + trace_api_bytes: worker.register_metric_context( + "trace_api_bytes".to_string(), + vec![], + MetricType::Distribution, + true, + MetricNamespace::Tracers, + ), trace_chunks_sent: worker.register_metric_context( "trace_chunks_sent".to_string(), vec![], diff --git a/tools/docker/Dockerfile.build b/tools/docker/Dockerfile.build index 58124a47c..4a493acbb 100644 --- a/tools/docker/Dockerfile.build +++ b/tools/docker/Dockerfile.build @@ -79,6 +79,7 @@ COPY "ddcommon/Cargo.toml" "ddcommon/" COPY "ddcommon-ffi/Cargo.toml" "ddcommon-ffi/" COPY "ddtelemetry/Cargo.toml" "ddtelemetry/" COPY "ddtelemetry-ffi/Cargo.toml" "ddtelemetry-ffi/" +COPY "ddsketch/Cargo.toml" "ddsketch/" COPY "profiling/Cargo.toml" "profiling/" COPY "profiling-ffi/Cargo.toml" "profiling-ffi/" COPY "profiling-replayer/Cargo.toml" "profiling-replayer/" diff --git a/trace-mini-agent/src/env_verifier.rs b/trace-mini-agent/src/env_verifier.rs index 714456de0..8363d224f 100644 --- a/trace-mini-agent/src/env_verifier.rs +++ b/trace-mini-agent/src/env_verifier.rs @@ -139,6 +139,9 @@ impl EnvVerifier for ServerlessEnvVerifier { .verify_gcp_environment_or_exit(verify_env_timeout) .await; } + trace_utils::EnvironmentType::LambdaFunction => { + trace_utils::MiniAgentMetadata::default() + } } } } diff --git a/trace-obfuscation/Cargo.toml b/trace-obfuscation/Cargo.toml index 53cc800d2..7f6cf98a2 100644 --- a/trace-obfuscation/Cargo.toml +++ b/trace-obfuscation/Cargo.toml @@ -21,7 +21,8 @@ ddcommon = { path = "../ddcommon" } [dev-dependencies] duplicate = "0.4.1" -criterion = "0.5" +criterion = { version = "0.5", features = [ "csv_output"] } +datadog-trace-utils = { path = "../trace-utils", features = ["test-utils"] } [[bench]] name = "trace_obfuscation" diff --git a/trace-obfuscation/benches/benchmarks/sql_obfuscation_bench.rs b/trace-obfuscation/benches/benchmarks/sql_obfuscation_bench.rs index f45c5d633..82be26811 100644 --- a/trace-obfuscation/benches/benchmarks/sql_obfuscation_bench.rs +++ b/trace-obfuscation/benches/benchmarks/sql_obfuscation_bench.rs @@ -1,18 +1,22 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 -use criterion::{black_box, criterion_group, Criterion}; +use criterion::{black_box, criterion_group, Criterion, SamplingMode}; use datadog_trace_obfuscation::sql::obfuscate_sql_string; fn sql_obfuscation(c: &mut Criterion) { let mut group = c.benchmark_group("sql"); - group.bench_function("obfsucate_sql_string", |b| { - b.iter(|| { - for (input, _) in CASES { - black_box(obfuscate_sql_string(input)); - } - }) - }); + // TODO https://datadoghq.atlassian.net/browse/APMSP-1228 + // Enable more sample modes for Criterion benchmarks + group + .sampling_mode(SamplingMode::Flat) + .bench_function("obfuscate_sql_string", |b| { + b.iter(|| { + for (input, _) in CASES { + black_box(obfuscate_sql_string(input)); + } + }) + }); } criterion_group!(benches, sql_obfuscation); diff --git a/trace-utils/Cargo.toml b/trace-utils/Cargo.toml index 4ef27bebf..a2e8ea640 100644 --- a/trace-utils/Cargo.toml +++ b/trace-utils/Cargo.toml @@ -25,6 +25,8 @@ rand = "0.8.5" bytes = "1.6.0" # This should only be used for testing. It isn't under dev-dependencies because test-utils can't be under #[cfg(test)]. httpmock = { version = "0.7.0", optional = true} +rmpv = { version = "1.3.0", default-features = false } +rmp = { version = "0.8.14", default-features = false } [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } @@ -32,4 +34,4 @@ serde_json = "1.0" httpmock = { version = "0.7.0"} [features] -test-utils = ["httpmock"] \ No newline at end of file +test-utils = ["httpmock"] diff --git a/trace-utils/src/config_utils.rs b/trace-utils/src/config_utils.rs index 6e7f4e533..3095e5800 100644 --- a/trace-utils/src/config_utils.rs +++ b/trace-utils/src/config_utils.rs @@ -10,6 +10,9 @@ const TRACE_INTAKE_ROUTE: &str = "/api/v0.2/traces"; const TRACE_STATS_INTAKE_ROUTE: &str = "/api/v0.2/stats"; pub fn read_cloud_env() -> Option<(String, trace_utils::EnvironmentType)> { + if let Ok(res) = env::var("AWS_LAMBDA_FUNCTION_NAME") { + return Some((res, trace_utils::EnvironmentType::LambdaFunction)); + } if let Ok(res) = env::var("K_SERVICE") { // Set by Google Cloud Functions for newer runtimes return Some((res, trace_utils::EnvironmentType::CloudFunction)); diff --git a/trace-utils/src/trace_utils.rs b/trace-utils/src/trace_utils.rs index dc308d24b..d560a35ac 100644 --- a/trace-utils/src/trace_utils.rs +++ b/trace-utils/src/trace_utils.rs @@ -1,8 +1,13 @@ // Copyright 2023-Present Datadog, Inc. https://www.datadoghq.com/ // SPDX-License-Identifier: Apache-2.0 +use anyhow::anyhow; +use bytes::buf::Reader; use hyper::{body::Buf, Body}; use log::{error, info}; +use rmp::decode::read_array_len; +use rmpv::decode::read_value; +use rmpv::{Integer, Value}; use std::cmp::Ordering; use std::collections::HashMap; @@ -20,6 +25,8 @@ const TOP_LEVEL_KEY: &str = "_top_level"; const TRACER_TOP_LEVEL_KEY: &str = "_dd.top_level"; const MAX_PAYLOAD_SIZE: usize = 50 * 1024 * 1024; +const MAX_STRING_DICT_SIZE: u32 = 25_000_000; +const SPAN_ELEMENT_COUNT: usize = 12; /// First value of returned tuple is the payload size pub async fn get_traces_from_request_body(body: Body) -> anyhow::Result<(usize, Vec>)> { @@ -40,6 +47,209 @@ pub async fn get_traces_from_request_body(body: Body) -> anyhow::Result<(usize, Ok((size, traces)) } +#[inline] +fn get_v05_strings_dict(reader: &mut Reader) -> anyhow::Result> { + let dict_size = + read_array_len(reader).map_err(|err| anyhow!("Error reading dict size: {err}"))?; + if dict_size > MAX_STRING_DICT_SIZE { + anyhow::bail!( + "Error deserializing strings dictionary. Dict size is too large: {dict_size}" + ); + } + let mut dict: Vec = Vec::with_capacity(dict_size.try_into()?); + for _ in 0..dict_size { + match read_value(reader)? { + Value::String(s) => { + let parsed_string = s.into_str().ok_or_else(|| anyhow!("Error reading string dict"))?; + dict.push(parsed_string); + } + val => anyhow::bail!("Error deserializing strings dictionary. Value in string dict is not a string: {val}") + } + } + Ok(dict) +} + +#[inline] +fn get_v05_span(reader: &mut Reader, dict: &[String]) -> anyhow::Result { + let mut span: Span = Default::default(); + let span_size = rmp::decode::read_array_len(reader) + .map_err(|err| anyhow!("Error reading span size: {err}"))? as usize; + if span_size != SPAN_ELEMENT_COUNT { + anyhow::bail!("Expected an array of exactly 12 elements in a span, got {span_size}"); + } + //0 - service + span.service = get_v05_string(reader, dict, "service")?; + // 1 - name + span.name = get_v05_string(reader, dict, "name")?; + // 2 - resource + span.resource = get_v05_string(reader, dict, "resource")?; + + // 3 - trace_id + match read_value(reader)? { + Value::Integer(i) => { + span.trace_id = i.as_u64().ok_or_else(|| { + anyhow!("Error reading span trace_id, value is not an integer: {i}") + })?; + } + val => anyhow::bail!("Error reading span trace_id, value is not an integer: {val}"), + }; + // 4 - span_id + match read_value(reader)? { + Value::Integer(i) => { + span.span_id = i.as_u64().ok_or_else(|| { + anyhow!("Error reading span span_id, value is not an integer: {i}") + })?; + } + val => anyhow::bail!("Error reading span span_id, value is not an integer: {val}"), + }; + // 5 - parent_id + match read_value(reader)? { + Value::Integer(i) => { + span.parent_id = i.as_u64().ok_or_else(|| { + anyhow!("Error reading span parent_id, value is not an integer: {i}") + })?; + } + val => anyhow::bail!("Error reading span parent_id, value is not an integer: {val}"), + }; + //6 - start + match read_value(reader)? { + Value::Integer(i) => { + span.start = i + .as_i64() + .ok_or_else(|| anyhow!("Error reading span start, value is not an integer: {i}"))?; + } + val => anyhow::bail!("Error reading span start, value is not an integer: {val}"), + }; + //7 - duration + match read_value(reader)? { + Value::Integer(i) => { + span.duration = i.as_i64().ok_or_else(|| { + anyhow!("Error reading span duration, value is not an integer: {i}") + })?; + } + val => anyhow::bail!("Error reading span duration, value is not an integer: {val}"), + }; + //8 - error + match read_value(reader)? { + Value::Integer(i) => { + span.error = i + .as_i64() + .ok_or_else(|| anyhow!("Error reading span error, value is not an integer: {i}"))? + as i32; + } + val => anyhow::bail!("Error reading span error, value is not an integer: {val}"), + } + //9 - meta + match read_value(reader)? { + Value::Map(meta) => { + for (k, v) in meta.iter() { + match k { + Value::Integer(k) => { + match v { + Value::Integer(v) => { + let key = str_from_dict(dict, *k)?; + let val = str_from_dict(dict, *v)?; + span.meta.insert(key, val); + } + _ => anyhow::bail!("Error reading span meta, value is not an integer and can't be looked up in dict: {v}") + } + } + _ => anyhow::bail!("Error reading span meta, key is not an integer and can't be looked up in dict: {k}") + } + } + } + val => anyhow::bail!("Error reading span meta, value is not a map: {val}"), + } + // 10 - metrics + match read_value(reader)? { + Value::Map(metrics) => { + for (k, v) in metrics.iter() { + match k { + Value::Integer(k) => { + match v { + Value::Integer(v) => { + let key = str_from_dict(dict, *k)?; + span.metrics.insert(key, v.as_f64().ok_or_else(||anyhow!("Error reading span metrics, value is not an integer: {v}"))?); + } + Value::F64(v) => { + let key = str_from_dict(dict, *k)?; + span.metrics.insert(key, *v); + } + _ => anyhow::bail!( + "Error reading span metrics, value is not a float or integer: {v}" + ), + } + } + _ => anyhow::bail!("Error reading span metrics, key is not an integer: {k}"), + } + } + } + val => anyhow::bail!("Error reading span metrics, value is not a map: {val}"), + } + + // 11 - type + match read_value(reader)? { + Value::Integer(s) => span.r#type = str_from_dict(dict, s)?, + val => anyhow::bail!("Error reading span type, value is not an integer: {val}"), + } + Ok(span) +} + +#[inline] +fn str_from_dict(dict: &[String], id: Integer) -> anyhow::Result { + let id = id + .as_i64() + .ok_or_else(|| anyhow!("Error reading string from dict, id is not an integer: {id}"))? + as usize; + if id >= dict.len() { + anyhow::bail!("Error reading string from dict, id out of bounds: {id}"); + } + Ok(dict[id].to_string()) +} + +#[inline] +fn get_v05_string( + reader: &mut Reader, + dict: &[String], + field_name: &str, +) -> anyhow::Result { + match read_value(reader)? { + Value::Integer(s) => { + str_from_dict(dict, s) + }, + val => anyhow::bail!("Error reading {field_name}, value is not an integer and can't be looked up in dict: {val}") + } +} + +pub async fn get_v05_traces_from_request_body( + body: Body, +) -> anyhow::Result<(usize, Vec>)> { + let buffer = hyper::body::aggregate(body).await?; + let body_size = buffer.remaining(); + let mut reader = buffer.reader(); + let wrapper_size = read_array_len(&mut reader)?; + if wrapper_size != 2 { + anyhow::bail!("Expected an arrary of exactly 2 elements, got {wrapper_size}"); + } + + let dict = get_v05_strings_dict(&mut reader)?; + + let traces_size = rmp::decode::read_array_len(&mut reader)?; + let mut traces: Vec> = Default::default(); + + for _ in 0..traces_size { + let spans_size = rmp::decode::read_array_len(&mut reader)?; + let mut trace: Vec = Default::default(); + + for _ in 0..spans_size { + let span = get_v05_span(&mut reader, &dict)?; + trace.push(span); + } + traces.push(trace); + } + Ok((body_size, traces)) +} + // Tags gathered from a trace's root span #[derive(Default)] pub struct RootSpanTags<'a> { @@ -217,6 +427,7 @@ pub fn set_serverless_root_span_tags( let origin_tag = match env_type { EnvironmentType::CloudFunction => "cloudfunction", EnvironmentType::AzureFunction => "azurefunction", + EnvironmentType::LambdaFunction => "lambda", // historical reasons }; span.meta .insert("_dd.origin".to_string(), origin_tag.to_string()); @@ -238,6 +449,7 @@ fn update_tracer_top_level(span: &mut Span) { pub enum EnvironmentType { CloudFunction, AzureFunction, + LambdaFunction, } #[derive(Clone, Default, Debug, Eq, PartialEq)] @@ -450,6 +662,88 @@ mod tests { assert!(coalesced.len() > 1 && coalesced.len() < 5); } + #[tokio::test] + #[allow(clippy::type_complexity)] + async fn get_v05_traces_from_request_body() { + let data: ( + Vec, + Vec< + Vec<( + u8, + u8, + u8, + u64, + u64, + u64, + i64, + i64, + i32, + HashMap, + HashMap, + u8, + )>, + >, + ) = ( + vec![ + "baggage".to_string(), + "item".to_string(), + "elasticsearch.version".to_string(), + "7.0".to_string(), + "my-name".to_string(), + "X".to_string(), + "my-service".to_string(), + "my-resource".to_string(), + "_dd.sampling_rate_whatever".to_string(), + "value whatever".to_string(), + "sql".to_string(), + ], + vec![vec![( + 6, + 4, + 7, + 1, + 2, + 3, + 123, + 456, + 1, + HashMap::from([(8, 9), (0, 1), (2, 3)]), + HashMap::from([(5, 1.2)]), + 10, + )]], + ); + let bytes = rmp_serde::to_vec(&data).unwrap(); + let res = + trace_utils::get_v05_traces_from_request_body(hyper::body::Body::from(bytes)).await; + assert!(res.is_ok()); + let (_, traces) = res.unwrap(); + let span = traces[0][0].clone(); + let test_span = Span { + service: "my-service".to_string(), + name: "my-name".to_string(), + resource: "my-resource".to_string(), + trace_id: 1, + span_id: 2, + parent_id: 3, + start: 123, + duration: 456, + error: 1, + meta: HashMap::from([ + ("baggage".to_string(), "item".to_string()), + ("elasticsearch.version".to_string(), "7.0".to_string()), + ( + "_dd.sampling_rate_whatever".to_string(), + "value whatever".to_string(), + ), + ]), + metrics: HashMap::from([("X".to_string(), 1.2)]), + meta_struct: HashMap::default(), + r#type: "sql".to_string(), + span_links: vec![], + }; + assert!(span == test_span); + } + #[tokio::test] #[cfg_attr(miri, ignore)] async fn test_get_traces_from_request_body() {