From 18cc292bd4debb6e68f2917a63a958d3dfa640cd Mon Sep 17 00:00:00 2001 From: Takuka0311 <1914426213@qq.com> Date: Mon, 18 Nov 2024 16:43:37 +0800 Subject: [PATCH 01/11] Remove LogFileProfiler (#1889) * init * polish --- core/app_config/AppConfig.cpp | 24 +- core/app_config/AppConfig.h | 6 +- core/application/Application.cpp | 34 +- .../common_provider/CommonConfigProvider.cpp | 10 +- .../LegacyCommonConfigProvider.cpp | 12 +- core/file_server/ConfigManager.cpp | 9 +- core/file_server/EventDispatcher.cpp | 2 +- core/file_server/EventDispatcher.h | 25 +- .../event_handler/EventHandler.cpp | 12 - core/file_server/reader/LogFileReader.cpp | 3 +- core/go_pipeline/LogtailPlugin.cpp | 7 +- core/logger/Logger.cpp | 7 +- core/monitor/AlarmManager.cpp | 11 +- core/monitor/LogFileProfiler.cpp | 562 ------------------ core/monitor/LogFileProfiler.h | 234 -------- core/monitor/MetricExportor.cpp | 5 +- core/monitor/Monitor.cpp | 66 +- core/monitor/Monitor.h | 15 +- .../monitor/metric_constants/AgentMetrics.cpp | 1 + .../metric_constants/MetricConstants.h | 1 + core/pipeline/PipelineContext.h | 5 +- core/plugin/flusher/sls/SLSClientManager.cpp | 8 +- .../processor/inner/ProcessorTagNative.cpp | 7 +- core/runner/ProcessorRunner.cpp | 1 - core/sls_control/SLSControl.cpp | 13 +- .../config/CommonConfigProviderUnittest.cpp | 9 +- .../unittest/config/ConfigUpdatorUnittest.cpp | 65 +- .../processor/ProcessorTagNativeUnittest.cpp | 7 +- core/unittest/sender/SenderUnittest.cpp | 61 +- 29 files changed, 193 insertions(+), 1029 deletions(-) delete mode 100644 core/monitor/LogFileProfiler.cpp delete mode 100644 core/monitor/LogFileProfiler.h diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index 777509210e..c154ce0c9b 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -30,7 +30,6 @@ #include "file_server/reader/LogFileReader.h" #include "json/value.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" #include "monitor/AlarmManager.h" #include "monitor/Monitor.h" #ifdef __ENTERPRISE__ @@ -433,14 +432,6 @@ string GetAgentSnapshotDir() { } } -string GetAgentProfileLogName() { - if (BOOL_FLAG(logtail_mode)) { - return "ilogtail_profile.LOG"; - } else { - return "loongcollector_profile.LOG"; - } -} - string GetAgentStatusLogName() { if (BOOL_FLAG(logtail_mode)) { return "ilogtail_status.LOG"; @@ -449,15 +440,6 @@ string GetAgentStatusLogName() { } } -string GetProfileSnapshotDumpFileName() { - if (BOOL_FLAG(logtail_mode)) { - return GetProcessExecutionDir() + STRING_FLAG(logtail_profile_snapshot); - } else { - return GetAgentLogDir() + "loongcollector_profile_snapshot"; - } -} - - string GetObserverEbpfHostPath() { if (BOOL_FLAG(logtail_mode)) { return STRING_FLAG(sls_observer_ebpf_host_path); @@ -884,14 +866,10 @@ void AppConfig::LoadResourceConf(const Json::Value& confJson) { "reader_close_unused_file_time", "ALIYUN_LOGTAIL_READER_CLOSE_UNUSED_FILE_TIME"); - if (confJson.isMember("log_profile_save_interval") && confJson["log_profile_save_interval"].isInt()) - LogFileProfiler::GetInstance()->SetProfileInterval(confJson["log_profile_save_interval"].asInt()); - LOG_DEBUG(sLogger, ("logreader delete interval", INT32_FLAG(logreader_filedeleted_remove_interval))( "check handler interval", INT32_FLAG(check_handler_timeout_interval))( - "reader close interval", INT32_FLAG(reader_close_unused_file_time))( - "profile interval", LogFileProfiler::GetInstance()->GetProfileInterval())); + "reader close interval", INT32_FLAG(reader_close_unused_file_time))); if (confJson.isMember("cpu_usage_limit")) { diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 22a5c6dd73..4a28cf7c1a 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -18,12 +18,12 @@ #include +#include #include #include #include #include #include -#include #include "InstanceConfig.h" #include "protobuf/sls/sls_logs.pb.h" @@ -49,9 +49,7 @@ std::string GetInotifyWatcherDirsDumpFileName(); std::string GetAgentLoggersPrefix(); std::string GetAgentLogName(); std::string GetAgentSnapshotDir(); -std::string GetAgentProfileLogName(); std::string GetAgentStatusLogName(); -std::string GetProfileSnapshotDumpFileName(); std::string GetObserverEbpfHostPath(); std::string GetSendBufferFileNamePrefix(); std::string GetLegacyUserLocalConfigFilePath(); @@ -301,7 +299,7 @@ class AppConfig { public: AppConfig(); - ~AppConfig(){}; + ~AppConfig() {}; void LoadInstanceConfig(const std::map>&); diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 3fcd0fabeb..6cef734be0 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -40,7 +40,6 @@ #include "file_server/event_handler/LogInput.h" #include "go_pipeline/LogtailPlugin.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" #include "monitor/MetricExportor.h" #include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" @@ -112,7 +111,7 @@ void Application::Init() { AppConfig::GetInstance()->LoadAppConfig(GetAgentConfigFile()); // Initialize basic information: IP, hostname, etc. - LogFileProfiler::GetInstance(); + LoongCollectorMonitor::GetInstance(); #ifdef __ENTERPRISE__ EnterpriseConfigProvider::GetInstance()->Init("enterprise"); EnterpriseConfigProvider::GetInstance()->LoadRegionConfig(); @@ -133,25 +132,25 @@ void Application::Init() { const string& interface = AppConfig::GetInstance()->GetBindInterface(); const string& configIP = AppConfig::GetInstance()->GetConfigIP(); if (!configIP.empty()) { - LogFileProfiler::mIpAddr = configIP; + LoongCollectorMonitor::mIpAddr = configIP; LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_ip", GetHostIp()); } else if (!interface.empty()) { - LogFileProfiler::mIpAddr = GetHostIp(interface); - if (LogFileProfiler::mIpAddr.empty()) { + LoongCollectorMonitor::mIpAddr = GetHostIp(interface); + if (LoongCollectorMonitor::mIpAddr.empty()) { LOG_WARNING(sLogger, ("failed to get ip from interface", "try to get any available ip")("interface", interface)); } - } else if (LogFileProfiler::mIpAddr.empty()) { + } else if (LoongCollectorMonitor::mIpAddr.empty()) { LOG_WARNING(sLogger, ("failed to get ip from hostname or eth0 or bond0", "try to get any available ip")); } - if (LogFileProfiler::mIpAddr.empty()) { - LogFileProfiler::mIpAddr = GetAnyAvailableIP(); - LOG_INFO(sLogger, ("get available ip succeeded", LogFileProfiler::mIpAddr)); + if (LoongCollectorMonitor::mIpAddr.empty()) { + LoongCollectorMonitor::mIpAddr = GetAnyAvailableIP(); + LOG_INFO(sLogger, ("get available ip succeeded", LoongCollectorMonitor::mIpAddr)); } const string& configHostName = AppConfig::GetInstance()->GetConfigHostName(); if (!configHostName.empty()) { - LogFileProfiler::mHostname = configHostName; + LoongCollectorMonitor::mHostname = configHostName; LogtailMonitor::GetInstance()->UpdateConstMetric("logtail_hostname", GetHostName()); } @@ -165,12 +164,12 @@ void Application::Init() { #endif int32_t systemBootTime = AppConfig::GetInstance()->GetSystemBootTime(); - LogFileProfiler::mSystemBootTime = systemBootTime > 0 ? systemBootTime : GetSystemBootTime(); + LoongCollectorMonitor::mSystemBootTime = systemBootTime > 0 ? systemBootTime : GetSystemBootTime(); // generate app_info.json Json::Value appInfoJson; - appInfoJson["ip"] = Json::Value(LogFileProfiler::mIpAddr); - appInfoJson["hostname"] = Json::Value(LogFileProfiler::mHostname); + appInfoJson["ip"] = Json::Value(LoongCollectorMonitor::mIpAddr); + appInfoJson["hostname"] = Json::Value(LoongCollectorMonitor::mHostname); appInfoJson["UUID"] = Json::Value(Application::GetInstance()->GetUUID()); appInfoJson["instance_id"] = Json::Value(Application::GetInstance()->GetInstanceId()); #ifdef __ENTERPRISE__ @@ -189,7 +188,7 @@ void Application::Init() { #define ILOGTAIL_COMPILER VERSION_STR(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) #endif appInfoJson["compiler"] = Json::Value(ILOGTAIL_COMPILER); - appInfoJson["os"] = Json::Value(LogFileProfiler::mOsDetail); + appInfoJson["os"] = Json::Value(LoongCollectorMonitor::mOsDetail); appInfoJson["update_time"] = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S"); string appInfo = appInfoJson.toStyledString(); OverwriteFile(GetAgentAppInfoFile(), appInfo); @@ -197,8 +196,8 @@ void Application::Init() { } void Application::Start() { // GCOVR_EXCL_START - LogFileProfiler::mStartTime = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S"); - LogtailMonitor::GetInstance()->UpdateConstMetric("start_time", LogFileProfiler::mStartTime); + LoongCollectorMonitor::mStartTime = GetTimeStamp(time(NULL), "%Y-%m-%d %H:%M:%S"); + LogtailMonitor::GetInstance()->UpdateConstMetric("start_time", LoongCollectorMonitor::mStartTime); #if defined(__ENTERPRISE__) && defined(_MSC_VER) InitWindowsSignalObject(); @@ -287,7 +286,6 @@ void Application::Start() { // GCOVR_EXCL_START lastConfigCheckTime = curTime; } if (curTime - lastProfilingCheckTime >= INT32_FLAG(profiling_check_interval)) { - LogFileProfiler::GetInstance()->SendProfileData(); MetricExportor::GetInstance()->PushMetrics(false); lastProfilingCheckTime = curTime; } @@ -330,7 +328,7 @@ void Application::Start() { // GCOVR_EXCL_START } // GCOVR_EXCL_STOP void Application::GenerateInstanceId() { - mInstanceId = CalculateRandomUUID() + "_" + LogFileProfiler::mIpAddr + "_" + ToString(mStartTime); + mInstanceId = CalculateRandomUUID() + "_" + LoongCollectorMonitor::mIpAddr + "_" + ToString(mStartTime); } bool Application::TryGetUUID() { diff --git a/core/config/common_provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp index 1a5edcfd31..58ae7a5340 100644 --- a/core/config/common_provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -22,7 +22,6 @@ #include "app_config/AppConfig.h" #include "application/Application.h" -#include "constants/Constants.h" #include "common/LogtailCommonFlags.h" #include "common/StringTools.h" #include "common/UUIDUtil.h" @@ -30,8 +29,9 @@ #include "common/version.h" #include "config/PipelineConfig.h" #include "config/feedbacker/ConfigFeedbackReceiver.h" +#include "constants/Constants.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" +#include "monitor/Monitor.h" #include "sdk/Common.h" #include "sdk/CurlImp.h" #include "sdk/Exception.h" @@ -206,11 +206,11 @@ string CommonConfigProvider::GetInstanceId() { } void CommonConfigProvider::FillAttributes(configserver::proto::v2::AgentAttributes& attributes) { - attributes.set_hostname(LogFileProfiler::mHostname); - attributes.set_ip(LogFileProfiler::mIpAddr); + attributes.set_hostname(LoongCollectorMonitor::mHostname); + attributes.set_ip(LoongCollectorMonitor::mIpAddr); attributes.set_version(ILOGTAIL_VERSION); google::protobuf::Map* extras = attributes.mutable_extras(); - extras->insert({"osDetail", LogFileProfiler::mOsDetail}); + extras->insert({"osDetail", LoongCollectorMonitor::mOsDetail}); } void addConfigInfoToRequest(const std::pair& configInfo, diff --git a/core/config/common_provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp index f5d000711f..b673bbeda5 100644 --- a/core/config/common_provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -26,7 +26,7 @@ #include "common/StringTools.h" #include "common/version.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" +#include "monitor/Monitor.h" #include "sdk/Common.h" #include "sdk/CurlImp.h" #include "sdk/Exception.h" @@ -112,7 +112,8 @@ void LegacyCommonConfigProvider::CheckUpdateThread() { } } -LegacyCommonConfigProvider::ConfigServerAddress LegacyCommonConfigProvider::GetOneConfigServerAddress(bool changeConfigServer) { +LegacyCommonConfigProvider::ConfigServerAddress +LegacyCommonConfigProvider::GetOneConfigServerAddress(bool changeConfigServer) { if (0 == mConfigServerAddresses.size()) { return ConfigServerAddress("", -1); // No address available } @@ -158,7 +159,7 @@ LegacyCommonConfigProvider::SendHeartbeat(const ConfigServerAddress& configServe heartBeatReq.set_agent_id(Application::GetInstance()->GetInstanceId()); heartBeatReq.set_agent_type("iLogtail"); attributes.set_version(ILOGTAIL_VERSION); - attributes.set_ip(LogFileProfiler::mIpAddr); + attributes.set_ip(LoongCollectorMonitor::mIpAddr); heartBeatReq.mutable_attributes()->MergeFrom(attributes); heartBeatReq.mutable_tags()->MergeFrom({GetConfigServerTags().begin(), GetConfigServerTags().end()}); heartBeatReq.set_running_status(""); @@ -289,8 +290,9 @@ void LegacyCommonConfigProvider::UpdateRemoteConfig( if (ec) { StopUsingConfigServer(); LOG_ERROR(sLogger, - ("failed to create dir for legacy common configs", "stop receiving config from legacy common config server")( - "dir", mPipelineSourceDir.string())("error code", ec.value())("error msg", ec.message())); + ("failed to create dir for legacy common configs", + "stop receiving config from legacy common config server")("dir", mPipelineSourceDir.string())( + "error code", ec.value())("error msg", ec.message())); return; } diff --git a/core/file_server/ConfigManager.cpp b/core/file_server/ConfigManager.cpp index 3ae6c241f9..089249921e 100644 --- a/core/file_server/ConfigManager.cpp +++ b/core/file_server/ConfigManager.cpp @@ -36,7 +36,6 @@ #include "app_config/AppConfig.h" #include "checkpoint/CheckPointManager.h" #include "common/CompressTools.h" -#include "constants/Constants.h" #include "common/ErrorUtil.h" #include "common/ExceptionBase.h" #include "common/FileSystemUtil.h" @@ -46,10 +45,10 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "common/version.h" +#include "constants/Constants.h" #include "file_server/EventDispatcher.h" -#include "file_server/event_handler/EventHandler.h" #include "file_server/FileServer.h" -#include "monitor/LogFileProfiler.h" +#include "file_server/event_handler/EventHandler.h" #include "monitor/AlarmManager.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineManager.h" @@ -101,7 +100,7 @@ DEFINE_FLAG_INT32(docker_config_update_interval, "interval between docker config namespace logtail { -// +// ParseConfResult ParseConfig(const std::string& configName, Json::Value& jsonRoot) { // Get full path, if it is a relative path, prepend process execution dir. std::string fullPath = configName; @@ -200,7 +199,7 @@ ConfigManager::ConfigManager() { // CorrectionLogtailSysConfDir(); // first create dir then rewrite system-uuid file in GetSystemUUID // use a thread to get uuid, work around for CalculateDmiUUID hang // mUUID = CalculateDmiUUID(); - // mInstanceId = CalculateRandomUUID() + "_" + LogFileProfiler::mIpAddr + "_" + ToString(time(NULL)); + // mInstanceId = CalculateRandomUUID() + "_" + LoongCollectorMonitor::mIpAddr + "_" + ToString(time(NULL)); // ReloadMappingConfig(); } diff --git a/core/file_server/EventDispatcher.cpp b/core/file_server/EventDispatcher.cpp index 1b03c06211..30350f126e 100644 --- a/core/file_server/EventDispatcher.cpp +++ b/core/file_server/EventDispatcher.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "EventDispatcher.h" + #include "Flags.h" #if defined(__linux__) #include @@ -45,7 +46,6 @@ #include "file_server/event_handler/LogInput.h" #include "file_server/polling/PollingDirFile.h" #include "file_server/polling/PollingModify.h" -#include "monitor/LogFileProfiler.h" #include "monitor/AlarmManager.h" #include "monitor/MetricExportor.h" #include "protobuf/sls/metric.pb.h" diff --git a/core/file_server/EventDispatcher.h b/core/file_server/EventDispatcher.h index d3162720a4..e1612254fb 100644 --- a/core/file_server/EventDispatcher.h +++ b/core/file_server/EventDispatcher.h @@ -17,24 +17,25 @@ #pragma once #include #if defined(__linux__) -#include -#include #include #include +#include +#include #include #endif #include #include + +#include #include #include #include -#include -#include "monitor/LogFileProfiler.h" -#include "file_server/polling/PollingModify.h" -#include "file_server/polling/PollingDirFile.h" -#include "file_server/event_listener/EventListener.h" + #include "checkpoint/CheckPointManager.h" #include "file_server/FileDiscoveryOptions.h" +#include "file_server/event_listener/EventListener.h" +#include "file_server/polling/PollingDirFile.h" +#include "file_server/polling/PollingModify.h" namespace logtail { class TimeoutHandler; @@ -184,11 +185,11 @@ class EventDispatcher { // * @return true on success; false on failure // */ // bool Dispatch(); -// #if defined(_MSC_VER) -// virtual void InitWindowsSignalObject() {} -// virtual void SyncWindowsSignalObject() {} -// virtual void ReleaseWindowsSignalObject() {} -// #endif + // #if defined(_MSC_VER) + // virtual void InitWindowsSignalObject() {} + // virtual void SyncWindowsSignalObject() {} + // virtual void ReleaseWindowsSignalObject() {} + // #endif // #if defined(__linux__) // virtual void InitShennong() = 0; // virtual void CheckShennong() = 0; diff --git a/core/file_server/event_handler/EventHandler.cpp b/core/file_server/event_handler/EventHandler.cpp index 44c4fa662f..8f3efe26ce 100644 --- a/core/file_server/event_handler/EventHandler.cpp +++ b/core/file_server/event_handler/EventHandler.cpp @@ -1071,18 +1071,6 @@ int32_t ModifyHandler::PushLogToProcessor(LogFileReaderPtr reader, LogBuffer* lo int32_t pushRetry = 0; if (!logBuffer->rawBuffer.empty()) { reader->ReportMetrics(logBuffer->readLength); - LogFileProfiler::GetInstance()->AddProfilingReadBytes(reader->GetConfigName(), - reader->GetRegion(), - reader->GetProject(), - reader->GetLogstore(), - reader->GetConvertedPath(), - reader->GetHostLogPath(), - reader->GetExtraTags(), - reader->GetDevInode().dev, - reader->GetDevInode().inode, - reader->GetFileSize(), - reader->GetLastFilePos(), - time(NULL)); PipelineEventGroup group = LogFileReader::GenerateEventGroup(reader, logBuffer); while (!ProcessorRunner::GetInstance()->PushQueue(reader->GetQueueKey(), 0, std::move(group))) // 10ms diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index 62c9a49c6f..08b7693354 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -46,7 +46,6 @@ #include "file_server/reader/GloablFileDescriptorManager.h" #include "file_server/reader/JsonLogFileReader.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" #include "monitor/AlarmManager.h" #include "monitor/metric_constants/MetricConstants.h" #include "pipeline/queue/ExactlyOnceQueueManager.h" @@ -296,7 +295,7 @@ bool LogFileReader::ShouldForceReleaseDeletedFileFd() { } void LogFileReader::InitReader(bool tailExisted, FileReadPolicy policy, uint32_t eoConcurrency) { - mSourceId = LogFileProfiler::mIpAddr + "_" + mReaderConfig.second->GetConfigName() + "_" + mHostLogPath + "_" + mSourceId = LoongCollectorMonitor::mIpAddr + "_" + mReaderConfig.second->GetConfigName() + "_" + mHostLogPath + "_" + CalculateRandomUUID(); if (!tailExisted) { diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index 993c050798..9ba4f91eb0 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -26,8 +26,8 @@ #include "container_manager/ConfigContainerInfoUpdateCmd.h" #include "file_server/ConfigManager.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" #include "monitor/AlarmManager.h" +#include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" #include "pipeline/queue/SenderQueueManager.h" #include "provider/Provider.h" @@ -66,8 +66,8 @@ LogtailPlugin::LogtailPlugin() { mPluginCfg["LoongcollectorLogDir"] = GetAgentLogDir(); mPluginCfg["LoongcollectorDataDir"] = GetAgentDataDir(); mPluginCfg["LoongcollectorThirdPartyDir"] = GetAgentThirdPartyDir(); - mPluginCfg["HostIP"] = LogFileProfiler::mIpAddr; - mPluginCfg["Hostname"] = LogFileProfiler::mHostname; + mPluginCfg["HostIP"] = LoongCollectorMonitor::mIpAddr; + mPluginCfg["Hostname"] = LoongCollectorMonitor::mHostname; mPluginCfg["EnableContainerdUpperDirDetect"] = BOOL_FLAG(enable_containerd_upper_dir_detect); mPluginCfg["EnableSlsMetricsFormat"] = BOOL_FLAG(enable_sls_metrics_format); } @@ -616,4 +616,3 @@ K8sContainerMeta LogtailPlugin::GetContainerMeta(const string& containerID) { } return K8sContainerMeta(); } - diff --git a/core/logger/Logger.cpp b/core/logger/Logger.cpp index c754d5b37b..84d03a375e 100644 --- a/core/logger/Logger.cpp +++ b/core/logger/Logger.cpp @@ -405,8 +405,8 @@ void Logger::LoadDefaultConfig(std::map& loggerCfgs, loggerCfgs.insert({DEFAULT_LOGGER_NAME, LoggerConfig{"AsyncFileSink", level::warn}}); if (sinkCfgs.find("AsyncFileSink") != sinkCfgs.end()) return; - sinkCfgs.insert({"AsyncFileSink", - SinkConfig{"AsyncFile", 10, 20000000, 300, GetAgentLogDir() + GetAgentLogName(), "Gzip"}}); + sinkCfgs.insert( + {"AsyncFileSink", SinkConfig{"AsyncFile", 10, 20000000, 300, GetAgentLogDir() + GetAgentLogName(), "Gzip"}}); } void Logger::LoadAllDefaultConfigs(std::map& loggerCfgs, @@ -414,15 +414,12 @@ void Logger::LoadAllDefaultConfigs(std::map& loggerCf LoadDefaultConfig(loggerCfgs, sinkCfgs); loggerCfgs.insert({GetAgentLoggersPrefix(), LoggerConfig{"AsyncFileSink", level::info}}); - loggerCfgs.insert({GetAgentLoggersPrefix() + "/profile", LoggerConfig{"AsyncFileSinkProfile", level::info}}); loggerCfgs.insert({GetAgentLoggersPrefix() + "/status", LoggerConfig{"AsyncFileSinkStatus", level::info}}); std::string dirPath = GetAgentSnapshotDir(); if (!Mkdir(dirPath)) { LogMsg(std::string("Create snapshot dir error ") + dirPath + ", error" + ErrnoToString(GetErrno())); } - sinkCfgs.insert( - {"AsyncFileSinkProfile", SinkConfig{"AsyncFile", 61, 1, 1, dirPath + PATH_SEPARATOR + GetAgentProfileLogName()}}); sinkCfgs.insert( {"AsyncFileSinkStatus", SinkConfig{"AsyncFile", 61, 1, 1, dirPath + PATH_SEPARATOR + GetAgentStatusLogName()}}); } diff --git a/core/monitor/AlarmManager.cpp b/core/monitor/AlarmManager.cpp index d4f0edfbf6..b6df20a6fc 100644 --- a/core/monitor/AlarmManager.cpp +++ b/core/monitor/AlarmManager.cpp @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "monitor/AlarmManager.h" +#include "AlarmManager.h" -#include "LogFileProfiler.h" +#include "Monitor.h" #include "app_config/AppConfig.h" #include "common/LogtailCommonFlags.h" #include "common/StringTools.h" @@ -217,7 +217,7 @@ void AlarmManager::SendAllRegionAlarm() { // LOG_DEBUG(sLogger, ("4Send Alarm", region)("region", sendRegionIndex)("alarm index", // mMessageType[sendAlarmTypeIndex])); - logGroup.set_source(LogFileProfiler::mIpAddr); + logGroup.set_source(LoongCollectorMonitor::mIpAddr); logGroup.set_category(ALARM_SLS_LOGSTORE_NAME); auto now = GetCurrentLogtailTime(); for (map::iterator mapIter = alarmMap.begin(); mapIter != alarmMap.end(); @@ -245,7 +245,7 @@ void AlarmManager::SendAllRegionAlarm() { contentPtr = logPtr->add_contents(); contentPtr->set_key("ip"); - contentPtr->set_value(LogFileProfiler::mIpAddr); + contentPtr->set_value(LoongCollectorMonitor::mIpAddr); contentPtr = logPtr->add_contents(); contentPtr->set_key("os"); @@ -319,8 +319,7 @@ void AlarmManager::SendAlarm(const AlarmType alarmType, string key = projectName + "_" + category; AlarmVector& alarmBufferVec = *MakesureLogtailAlarmMapVecUnlocked(region); if (alarmBufferVec[alarmType].find(key) == alarmBufferVec[alarmType].end()) { - AlarmMessage* messagePtr - = new AlarmMessage(mMessageType[alarmType], projectName, category, message, 1); + AlarmMessage* messagePtr = new AlarmMessage(mMessageType[alarmType], projectName, category, message, 1); alarmBufferVec[alarmType].insert(pair(key, messagePtr)); } else alarmBufferVec[alarmType][key]->IncCount(); diff --git a/core/monitor/LogFileProfiler.cpp b/core/monitor/LogFileProfiler.cpp deleted file mode 100644 index 95b9456fb7..0000000000 --- a/core/monitor/LogFileProfiler.cpp +++ /dev/null @@ -1,562 +0,0 @@ -// Copyright 2022 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "LogFileProfiler.h" - -#include - -#include "app_config/AppConfig.h" -#include "constants/Constants.h" -#include "common/ErrorUtil.h" -#include "common/LogtailCommonFlags.h" -#include "common/MachineInfoUtil.h" -#include "common/RuntimeUtil.h" -#include "common/StringTools.h" -#include "common/TimeUtil.h" -#include "common/version.h" -#include "file_server/ConfigManager.h" -#include "logger/Logger.h" -#include "provider/Provider.h" -#include "pipeline/queue/QueueKeyManager.h" - -DEFINE_FLAG_INT32(profile_data_send_interval, "interval of send LogFile/DomainSocket profile data, seconds", 600); - -using namespace std; -using namespace sls_logs; - -namespace logtail { - -string LogFileProfiler::mHostname; -string LogFileProfiler::mIpAddr; -string LogFileProfiler::mOsDetail; -string LogFileProfiler::mUsername; -int32_t LogFileProfiler::mSystemBootTime = -1; -string LogFileProfiler::mECSInstanceID; -string LogFileProfiler::mECSUserID; -string LogFileProfiler::mECSRegionID; -string LogFileProfiler::mStartTime; - -LogFileProfiler::LogFileProfiler() { - srand(time(NULL)); - mSendInterval = INT32_FLAG(profile_data_send_interval); - mLastSendTime = time(NULL) - (rand() % (mSendInterval / 10)) * 10; - mDumpFileName = GetProfileSnapshotDumpFileName(); - mBakDumpFileName = GetProfileSnapshotDumpFileName() + "_bak"; - - mHostname = GetHostName(); -#if defined(_MSC_VER) - mHostname = EncodingConverter::GetInstance()->FromACPToUTF8(mHostname); -#endif - mIpAddr = GetHostIp(); - mOsDetail = GetOsDetail(); - mUsername = GetUsername(); - // TODO: This may take up to 3s to construct the object. This is bad. - ECSMeta ecsMeta = FetchECSMeta(); - mECSInstanceID = ecsMeta.instanceID; - mECSUserID = ecsMeta.userID; - mECSRegionID = ecsMeta.regionID; -} - -bool LogFileProfiler::GetProfileData(LogGroup& logGroup, LogStoreStatistic* statistic) { - if (statistic->mReadBytes + statistic->mSkipBytes == 0) - return false; - - Log* logPtr = logGroup.add_logs(); - auto now = GetCurrentLogtailTime(); - SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec); - - Log_Content* contentPtr = logPtr->add_contents(); - contentPtr->set_key("logreader_project_name"); - contentPtr->set_value(statistic->mProjectName); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("category"); - contentPtr->set_value(statistic->mCategory); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("config_name"); - contentPtr->set_value(statistic->mConfigName); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("file_name"); - contentPtr->set_value(statistic->mConvertedPath.empty() ? "logstore_statistics" : statistic->mConvertedPath); - if (!statistic->mHostLogPath.empty()) { - contentPtr = logPtr->add_contents(); - contentPtr->set_key("host_log_path"); - contentPtr->set_value(statistic->mHostLogPath); - } - contentPtr = logPtr->add_contents(); - contentPtr->set_key("loongcollector_version"); - contentPtr->set_value(ILOGTAIL_VERSION); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("source_ip"); - contentPtr->set_value(mIpAddr); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("os"); - contentPtr->set_value(OS_NAME); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("total_bytes"); - contentPtr->set_value(ToString(statistic->mReadBytes)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("skip_bytes"); - contentPtr->set_value(ToString(statistic->mSkipBytes)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("succeed_lines"); - contentPtr->set_value(ToString(statistic->mSplitLines - statistic->mParseFailures - statistic->mSendFailures)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("parse_failures"); - contentPtr->set_value(ToString(statistic->mParseFailures)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("send_failures"); - contentPtr->set_value(ToString(statistic->mSendFailures)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("regex_match_failures"); - contentPtr->set_value(ToString(statistic->mRegexMatchFailures)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("history_data_failures"); - contentPtr->set_value(ToString(statistic->mHistoryFailures)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("time_format_failures"); - contentPtr->set_value(ToString(statistic->mParseTimeFailures)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("file_dev"); - contentPtr->set_value(ToString(statistic->mFileDev)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("file_inode"); - contentPtr->set_value(ToString(statistic->mFileInode)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("last_read_time"); - contentPtr->set_value(ToString(statistic->mLastReadTime)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("read_count"); - contentPtr->set_value(ToString(statistic->mReadCount)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("file_size"); - contentPtr->set_value(ToString(statistic->mFileSize)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("read_offset"); - contentPtr->set_value(ToString(statistic->mReadOffset)); - contentPtr = logPtr->add_contents(); - contentPtr->set_key("read_avg_delay"); - contentPtr->set_value(ToString(statistic->mReadCount == 0 ? 0 : statistic->mReadDelaySum / statistic->mReadCount)); - - if (!statistic->mTags.empty()) { - const std::vector& extraTags = statistic->mTags; - for (size_t i = 0; i < extraTags.size(); ++i) { - contentPtr = logPtr->add_contents(); - contentPtr->set_key(extraTags[i].key()); - contentPtr->set_value(extraTags[i].value()); - } - } - - if (!statistic->mErrorLine.empty()) { - contentPtr = logPtr->add_contents(); - contentPtr->set_key("error_line"); - contentPtr->set_value(statistic->mErrorLine); - } - - // get logstore send info - // if (statistic->mHostLogPath.empty()) { - // QueueKey fbKey = QueueKeyManager::GetInstance()->GetKey("-flusher_sls-" + statistic->mProjectName + "#" - // + statistic->mCategory); - // LogstoreSenderStatistics senderStatistics = Sender::GetInstance()->GetSenderStatistics(fbKey); - - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("max_unsend_time"); - // contentPtr->set_value(ToString(senderStatistics.mMaxUnsendTime)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("min_unsend_time"); - // contentPtr->set_value(ToString(senderStatistics.mMinUnsendTime)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("max_send_success_time"); - // contentPtr->set_value(ToString(senderStatistics.mMaxSendSuccessTime)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("send_queue_size"); - // contentPtr->set_value(ToString(senderStatistics.mSendQueueSize)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("send_network_error"); - // contentPtr->set_value(ToString(senderStatistics.mSendNetWorkErrorCount)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("send_quota_error"); - // contentPtr->set_value(ToString(senderStatistics.mSendQuotaErrorCount)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("send_discard_error"); - // contentPtr->set_value(ToString(senderStatistics.mSendDiscardErrorCount)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("send_success_count"); - // contentPtr->set_value(ToString(senderStatistics.mSendSuccessCount)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("send_block_flag"); - // contentPtr->set_value(ToString(senderStatistics.mSendBlockFlag)); - // contentPtr = logPtr->add_contents(); - // contentPtr->set_key("sender_valid_flag"); - // contentPtr->set_value(ToString(senderStatistics.mValidToSendFlag)); - // } - - return true; -} - -LogFileProfiler::LogstoreSenderStatisticsMap* -LogFileProfiler::MakesureRegionStatisticsMapUnlocked(const string& region) { - // @todo - // string region; - std::map::iterator iter = mAllStatisticsMap.find(region); - if (iter != mAllStatisticsMap.end()) { - return iter->second; - } - LogstoreSenderStatisticsMap* pMap = new LogstoreSenderStatisticsMap; - mAllStatisticsMap.insert(std::pair(region, pMap)); - return pMap; -} - -void LogFileProfiler::SendProfileData(bool forceSend) { - int32_t curTime = time(NULL); - if (!forceSend && (curTime - mLastSendTime < mSendInterval)) - return; - size_t sendRegionIndex = 0; - Json::Value detail; - Json::Value logstore; - do { - LogGroup logGroup; - logGroup.set_category("shennong_log_profile"); - logGroup.set_source(LogFileProfiler::mIpAddr); - string region; - { - // only lock statisticsMap, not sender or dump - std::lock_guard lock(mStatisticLock); - if (mAllStatisticsMap.size() <= sendRegionIndex) { - break; - } - - size_t iterIndex = 0; - std::map::iterator iter = mAllStatisticsMap.begin(); - while (iterIndex != sendRegionIndex) { - ++iterIndex; - ++iter; - } - - ++sendRegionIndex; - region = iter->first; - LogstoreSenderStatisticsMap& statisticsMap = *(iter->second); - if (statisticsMap.size() > (size_t)0) { - std::unordered_map::iterator iter = statisticsMap.begin(); - for (; iter != statisticsMap.end();) { - GetProfileData(logGroup, iter->second); - if ((curTime - iter->second->mLastUpdateTime) > mSendInterval * 3) { - delete iter->second; - iter = statisticsMap.erase(iter); - } else { - iter->second->Reset(); - iter++; - } - } - } - } - UpdateDumpData(logGroup, detail, logstore); - GetProfileSender()->SendToProfileProject(region, logGroup); - } while (true); - DumpToLocal(curTime, forceSend, detail, logstore); - mLastSendTime = curTime; -} - - -// 1. when in container, convertedPath is the file path in container, hostLogPath is the file path on host. -// eg. /home/admin/access.log in container, convertedPath = "/home/admin/access.log", -// hostLogPath="/logtail_host/xxx/home/admin/access.log". so hostLogPath is unique. -// 2. On host, convertedPath = hostLogPath. -void LogFileProfiler::AddProfilingData(const std::string& configName, - const std::string& region, - const std::string& projectName, - const std::string& category, - const std::string& convertedPath, - const std::string& hostLogPath, - const std::vector& tags, - uint64_t readBytes, - uint64_t skipBytes, - uint64_t splitLines, - uint64_t parseFailures, - uint64_t regexMatchFailures, - uint64_t parseTimeFailures, - uint64_t historyFailures, - uint64_t sendFailures, - const std::string& errorLine) { - if (!hostLogPath.empty()) { - // logstore statistics - AddProfilingData(configName, - region, - projectName, - category, - "", - "", - tags, - readBytes, - skipBytes, - splitLines, - parseFailures, - regexMatchFailures, - parseTimeFailures, - historyFailures, - sendFailures, - ""); - } - string key = projectName + "_" + category + "_" + configName + "_" + hostLogPath; - std::lock_guard lock(mStatisticLock); - LogstoreSenderStatisticsMap& statisticsMap = *MakesureRegionStatisticsMapUnlocked(region); - std::unordered_map::iterator iter = statisticsMap.find(key); - if (iter != statisticsMap.end()) { - (iter->second)->mReadBytes += readBytes; - (iter->second)->mSkipBytes += skipBytes; - (iter->second)->mSplitLines += splitLines; - (iter->second)->mParseFailures += parseFailures; - (iter->second)->mRegexMatchFailures += regexMatchFailures; - (iter->second)->mParseTimeFailures += parseTimeFailures; - (iter->second)->mHistoryFailures += historyFailures; - (iter->second)->mSendFailures += sendFailures; - if ((iter->second)->mErrorLine.empty()) - (iter->second)->mErrorLine = errorLine; - (iter->second)->mLastUpdateTime = time(NULL); - } else { - LogStoreStatistic* statistic = NULL; - if (hostLogPath.empty()) { - std::vector empty; - statistic = new LogStoreStatistic(configName, - projectName, - category, - convertedPath, - hostLogPath, - empty, - readBytes, - skipBytes, - splitLines, - parseFailures, - regexMatchFailures, - parseTimeFailures, - historyFailures, - sendFailures, - errorLine); - } else { - statistic = new LogStoreStatistic(configName, - projectName, - category, - convertedPath, - hostLogPath, - tags, - readBytes, - skipBytes, - splitLines, - parseFailures, - regexMatchFailures, - parseTimeFailures, - historyFailures, - sendFailures, - errorLine); - } - statisticsMap.insert(std::pair(key, statistic)); - } -} - -void LogFileProfiler::AddProfilingSkipBytes(const std::string& configName, - const std::string& region, - const std::string& projectName, - const std::string& category, - const std::string& convertedPath, - const std::string& hostLogPath, - const std::vector& tags, - uint64_t skipBytes) { - if (!hostLogPath.empty()) { - // logstore statistics - AddProfilingSkipBytes(configName, region, projectName, category, "", "", tags, skipBytes); - } - string key = projectName + "_" + category + "_" + configName + "_" + hostLogPath; - std::lock_guard lock(mStatisticLock); - LogstoreSenderStatisticsMap& statisticsMap = *MakesureRegionStatisticsMapUnlocked(region); - std::unordered_map::iterator iter = statisticsMap.find(key); - if (iter != statisticsMap.end()) { - (iter->second)->mSkipBytes += skipBytes; - (iter->second)->mLastUpdateTime = time(NULL); - } else { - LogStoreStatistic* statistic = NULL; - if (hostLogPath.empty()) { - std::vector empty; - statistic = new LogStoreStatistic(configName, projectName, category, convertedPath, hostLogPath, empty); - } else { - statistic = new LogStoreStatistic(configName, projectName, category, convertedPath, hostLogPath, tags); - } - statistic->mSkipBytes += skipBytes; - statisticsMap.insert(std::pair(key, statistic)); - } -} - -void LogFileProfiler::AddProfilingReadBytes(const std::string& configName, - const std::string& region, - const std::string& projectName, - const std::string& category, - const std::string& convertedPath, - const std::string& hostLogPath, - const std::vector& tags, - uint64_t dev, - uint64_t inode, - uint64_t fileSize, - uint64_t readOffset, - int32_t lastReadTime) { - if (!hostLogPath.empty()) { - // logstore statistics - AddProfilingReadBytes( - configName, region, projectName, category, "", "", tags, dev, inode, fileSize, readOffset, lastReadTime); - } - string key = projectName + "_" + category + "_" + configName + "_" + hostLogPath; - std::lock_guard lock(mStatisticLock); - LogstoreSenderStatisticsMap& statisticsMap = *MakesureRegionStatisticsMapUnlocked(region); - std::unordered_map::iterator iter = statisticsMap.find(key); - if (iter != statisticsMap.end()) { - (iter->second)->UpdateReadInfo(dev, inode, fileSize, readOffset, lastReadTime); - } else { - LogStoreStatistic* statistic = NULL; - if (hostLogPath.empty()) { - std::vector empty; - statistic = new LogStoreStatistic(configName, projectName, category, convertedPath, hostLogPath, empty); - } else { - statistic = new LogStoreStatistic(configName, projectName, category, convertedPath, hostLogPath, tags); - } - statistic->UpdateReadInfo(dev, inode, fileSize, readOffset, lastReadTime); - statisticsMap.insert(std::pair(key, statistic)); - } -} - -void LogFileProfiler::DumpToLocal(int32_t curTime, bool forceSend, Json::Value& detail, Json::Value& logstore) { - Json::Value root; - root["version"] = ILOGTAIL_VERSION; - root["ip"] = mIpAddr; - root["begin_time"] = (Json::UInt64)mLastSendTime; - root["begin_time_readable"] = GetTimeStamp(mLastSendTime, "%Y-%m-%d %H:%M:%S"); - root["end_time"] = (Json::UInt64)curTime; - root["end_time_readable"] = GetTimeStamp(curTime, "%Y-%m-%d %H:%M:%S"); - - - root["detail"] = detail; - root["logstore"] = logstore; - string styledRoot = root.toStyledString(); - if (forceSend) { - FILE* pFile = fopen(mBakDumpFileName.c_str(), "w"); - if (pFile == NULL) { - LOG_ERROR(sLogger, ("open file failed", mBakDumpFileName)("errno", errno)); - return; - } - fwrite(styledRoot.c_str(), 1, styledRoot.size(), pFile); - fclose(pFile); -#if defined(_MSC_VER) - remove(mDumpFileName.c_str()); -#endif - if (rename(mBakDumpFileName.c_str(), mDumpFileName.c_str()) == -1) - LOG_INFO(sLogger, - ("rename profile snapshot fail, file", mDumpFileName)("error", ErrnoToString(GetErrno()))); - } - - static auto gProfileLogger = Logger::Instance().GetLogger(GetAgentLoggersPrefix() + "/profile"); - LOG_INFO(gProfileLogger, ("\n", styledRoot)); -} - -void LogFileProfiler::UpdateDumpData(const sls_logs::LogGroup& logGroup, Json::Value& detail, Json::Value& logstore) { - for (int32_t logIdx = 0; logIdx < logGroup.logs_size(); ++logIdx) { - Json::Value category; - const Log& log = logGroup.logs(logIdx); - bool logstoreFlag = false; - for (int32_t conIdx = 0; conIdx < log.contents_size(); ++conIdx) { - const Log_Content& content = log.contents(conIdx); - const string& key = content.key(); - const string& value = content.value(); - if (key == "logreader_project_name") - category["project"] = value; - else if (key == "category") - category["logstore"] = value; - else if (key == "config_name") - category["config_name"] = value; - else if (key == "file_name") { - category["file"] = value; - if (value == "logstore_statistics") { - logstoreFlag = true; - } - } else if (key == "total_bytes") - category["read_bytes"] = value; - else if (key == "skip_bytes") - category["skip_bytes"] = value; - else if (key == "succeed_lines") - category["split_lines"] = value; - else if (key == "parse_failures") - category["parse_fail_lines"] = value; - else if (key == "file_dev") - category["file_dev"] = value; - else if (key == "file_inode") - category["file_inode"] = value; - else if (key == "last_read_time") - category["last_read_time"] = value; - else if (key == "read_count") - category["read_count"] = value; - else if (key == "file_size") - category["file_size"] = value; - else if (key == "read_offset") - category["read_offset"] = value; - else if (key == "read_avg_delay") - category["read_avg_delay"] = value; - else if (key == "max_unsend_time") - category["max_unsend_time"] = value; - else if (key == "min_unsend_time") - category["min_unsend_time"] = value; - else if (key == "max_send_success_time") - category["max_send_success_time"] = value; - else if (key == "send_queue_size") - category["send_queue_size"] = value; - else if (key == "send_network_error") - category["send_network_error"] = value; - else if (key == "send_quota_error") - category["send_quota_error"] = value; - else if (key == "send_discard_error") - category["send_discard_error"] = value; - else if (key == "send_success_count") - category["send_success_count"] = value; - else if (key == "sender_valid_flag") - category["sender_valid_flag"] = value; - else if (key == "send_block_flag") - category["send_block_flag"] = value; - } - if (logstoreFlag) { - logstore.append(category); - } else { - detail.append(category); - } - } -} - -#ifdef APSARA_UNIT_TEST_MAIN -uint64_t LogFileProfiler::GetProfilingLines(const std::string& projectName, - const std::string& category, - const std::string& convertedPath) { - std::string key = projectName + "_" + category + "_" + convertedPath; - std::lock_guard lock(mStatisticLock); - if (mAllStatisticsMap.size() != (size_t)1) { - return 0; - } - LogstoreSenderStatisticsMap statisticMap = *(mAllStatisticsMap.begin()->second); - std::unordered_map::iterator iter = statisticMap.find(key); - if (iter == statisticMap.end()) - return 0; - else - return (iter->second->mSplitLines - iter->second->mParseFailures); -} - -void LogFileProfiler::CleanEnviroments() { - std::lock_guard lock(mStatisticLock); - // just for test, memory leaks - mAllStatisticsMap.clear(); -} -#endif - -} // namespace logtail diff --git a/core/monitor/LogFileProfiler.h b/core/monitor/LogFileProfiler.h deleted file mode 100644 index f9ecc4cce7..0000000000 --- a/core/monitor/LogFileProfiler.h +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright 2022 iLogtail Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include -#include -#include -#include -#include -#include "protobuf/sls/sls_logs.pb.h" - -namespace sls_logs { -class LogGroup; -} - -namespace logtail { -// Collect the log file's profile such as lines processed. -class LogFileProfiler { -public: - static LogFileProfiler* GetInstance() { - static LogFileProfiler* ptr = new LogFileProfiler(); - return ptr; - } - - void AddProfilingData(const std::string& configName, - const std::string& region, - const std::string& projectName, - const std::string& category, - const std::string& convertedPath, - const std::string& hostLogPath, - const std::vector& tags, - uint64_t readBytes, - uint64_t skipBytes, - uint64_t splitLines, - uint64_t parseFailures, - uint64_t regexMatchFailures, - uint64_t parseTimeFailures, - uint64_t historyFailures, - uint64_t sendFailures, - const std::string& errorLine); - void AddProfilingSkipBytes(const std::string& configName, - const std::string& region, - const std::string& projectName, - const std::string& category, - const std::string& convertedPath, - const std::string& hostLogPath, - const std::vector& tags, - uint64_t skipBytes); - - void AddProfilingReadBytes(const std::string& configName, - const std::string& region, - const std::string& projectName, - const std::string& category, - const std::string& convertedPath, - const std::string& host, - const std::vector& tags, - uint64_t dev, - uint64_t inode, - uint64_t fileSize, - uint64_t readOffset, - int32_t lastReadTime); - - void SendProfileData(bool forceSend = false); - - void SetProfileInterval(int32_t interval) { mSendInterval = interval; } - - int32_t GetProfileInterval() { return mSendInterval; } - - static std::string mHostname; - static std::string mIpAddr; - static std::string mOsDetail; - static std::string mUsername; - static int32_t mSystemBootTime; - static std::string mECSInstanceID; - static std::string mECSUserID; - static std::string mECSRegionID; - static std::string mStartTime; - -private: - struct LogStoreStatistic { - LogStoreStatistic(const std::string& configName, - const std::string& projectName, - const std::string& category, - const std::string& convertedPath, - const std::string& hostLogPath, - const std::vector& tags, - uint64_t readBytes = 0, - uint64_t skipBytes = 0, - uint64_t splitLines = 0, - uint64_t parseFailures = 0, - uint64_t regexMatchFailures = 0, - uint64_t parseTimeFailures = 0, - uint64_t historyFailures = 0, - uint64_t sendFailures = 0, - const std::string& errorLine = "") - : mConfigName(configName), - mProjectName(projectName), - mCategory(category), - mConvertedPath(convertedPath), - mHostLogPath(hostLogPath), - mTags(tags), - mReadBytes(readBytes), - mSkipBytes(skipBytes), - mSplitLines(splitLines), - mParseFailures(parseFailures), - mRegexMatchFailures(regexMatchFailures), - mParseTimeFailures(parseTimeFailures), - mHistoryFailures(historyFailures), - mSendFailures(sendFailures), - mErrorLine(errorLine) { - mLastUpdateTime = time(NULL); - mFileDev = 0; - mFileInode = 0; - mFileSize = 0; - mReadOffset = 0; - mLastReadTime = 0; - mReadCount = 0; - mReadDelaySum = 0; - } - - void Reset() { - mFileDev = 0; - mFileInode = 0; - mFileSize = 0; - mReadOffset = 0; - mLastReadTime = 0; - mReadCount = 0; - mReadDelaySum = 0; - mReadBytes = 0; - mSkipBytes = 0; - mSplitLines = 0; - mParseFailures = 0; - mRegexMatchFailures = 0; - mParseTimeFailures = 0; - mHistoryFailures = 0; - mSendFailures = 0; - mErrorLine.clear(); - } - - void - UpdateReadInfo(uint64_t dev, uint64_t inode, uint64_t fileSize, uint64_t readOffset, int32_t lastReadTime) { - mFileDev = dev; - mFileInode = inode; - mFileSize = fileSize; - mReadOffset = readOffset; - mLastReadTime = lastReadTime; - ++mReadCount; - mReadDelaySum += fileSize > readOffset ? fileSize - readOffset : 0; - } - - std::string mConfigName; - std::string mProjectName; - std::string mCategory; - std::string mConvertedPath; - std::string mHostLogPath; - std::vector mTags; - // how many bytes processed - uint64_t mReadBytes; - // how many bytes skiped - uint64_t mSkipBytes; - // how many lines processed: mSplitLines - // how many lines parse failed: mParseFailures - // how many lines send failed: mSendFailures - // how many lines succeed send: mSplitLines - mParseFailures - mSendFailures - uint64_t mSplitLines; - // how many lines parse fails (include all failures) - uint64_t mParseFailures; - // how many lines regex match fail(include boost crash or not match) - uint64_t mRegexMatchFailures; - // how many lines parse timeformat fail - uint64_t mParseTimeFailures; - // how many lines history data discarded - uint64_t mHistoryFailures; - // how many lines send fails - uint64_t mSendFailures; - // one sample error line - std::string mErrorLine; - int32_t mLastUpdateTime; - - uint64_t mFileDev; - uint64_t mFileInode; - uint64_t mFileSize; - uint64_t mReadOffset; - int32_t mLastReadTime; - // ++mReadCount every call - uint32_t mReadCount; - // mReadDelaySum += mFileSize - mReadOffset every call - // then average delay is mReadDelaySum / mReadCount - uint64_t mReadDelaySum; - }; - - std::string mDumpFileName; - std::string mBakDumpFileName; - int32_t mLastSendTime; - int32_t mSendInterval; - // key : "project_name" + "_" + "category" + "_" + "filename" - typedef std::unordered_map LogstoreSenderStatisticsMap; - // key : region, value :unordered_map - std::map mAllStatisticsMap; - std::mutex mStatisticLock; - - LogFileProfiler(); - ~LogFileProfiler() {} - void DumpToLocal(int32_t curTime, bool forceSend, Json::Value& detail, Json::Value& logstore); - void UpdateDumpData(const sls_logs::LogGroup& logGroup, Json::Value& detail, Json::Value& logstore); - bool GetProfileData(sls_logs::LogGroup& logGroup, LogStoreStatistic* statistic); - - LogstoreSenderStatisticsMap* MakesureRegionStatisticsMapUnlocked(const std::string& region); - -#ifdef APSARA_UNIT_TEST_MAIN - friend class EventDispatcherTest; - friend class SenderUnittest; - - uint64_t - GetProfilingLines(const std::string& projectName, const std::string& category, const std::string& filename); - void CleanEnviroments(); -#endif -}; - -} // namespace logtail diff --git a/core/monitor/MetricExportor.cpp b/core/monitor/MetricExportor.cpp index 2440fe2106..d6417bdc00 100644 --- a/core/monitor/MetricExportor.cpp +++ b/core/monitor/MetricExportor.cpp @@ -16,9 +16,8 @@ #include -#include "LogFileProfiler.h" -#include "MetricManager.h" #include "MetricConstants.h" +#include "MetricManager.h" #include "app_config/AppConfig.h" #include "common/FileSystemUtil.h" #include "common/RuntimeUtil.h" @@ -88,7 +87,7 @@ void MetricExportor::SendToSLS(std::map& logGr for (iter = logGroupMap.begin(); iter != logGroupMap.end(); iter++) { sls_logs::LogGroup* logGroup = iter->second; logGroup->set_category(METRIC_SLS_LOGSTORE_NAME); - logGroup->set_source(LogFileProfiler::mIpAddr); + logGroup->set_source(LoongCollectorMonitor::mIpAddr); logGroup->set_topic(METRIC_TOPIC_TYPE); if (METRIC_REGION_DEFAULT == iter->first) { GetProfileSender()->SendToProfileProject(GetProfileSender()->GetDefaultProfileRegion(), *logGroup); diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index e09ba3e0b0..f83f9f37aa 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -23,7 +23,7 @@ #include #include "app_config/AppConfig.h" -#include "constants/Constants.h" +#include "application/Application.h" #include "common/DevInode.h" #include "common/ExceptionBase.h" #include "common/LogtailCommonFlags.h" @@ -32,16 +32,15 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "common/version.h" +#include "constants/Constants.h" #include "file_server/event_handler/LogInput.h" #include "go_pipeline/LogtailPlugin.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" #include "monitor/AlarmManager.h" #include "monitor/MetricExportor.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "protobuf/sls/sls_logs.pb.h" #include "runner/FlusherRunner.h" -#include "application/Application.h" #include "sdk/Common.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" @@ -58,6 +57,16 @@ DECLARE_FLAG_BOOL(check_profile_region); namespace logtail { +string LoongCollectorMonitor::mHostname; +string LoongCollectorMonitor::mIpAddr; +string LoongCollectorMonitor::mOsDetail; +string LoongCollectorMonitor::mUsername; +int32_t LoongCollectorMonitor::mSystemBootTime = -1; +string LoongCollectorMonitor::mECSInstanceID; +string LoongCollectorMonitor::mECSUserID; +string LoongCollectorMonitor::mECSRegionID; +string LoongCollectorMonitor::mStartTime; + inline void CpuStat::Reset() { #if defined(__linux__) mUserTime = 0; @@ -248,13 +257,13 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { } // the unique id of current instance - std::string id = sdk::Base64Enconde(LogFileProfiler::mHostname + LogFileProfiler::mIpAddr + ILOGTAIL_VERSION - + GetProcessExecutionDir()); + std::string id = sdk::Base64Enconde(LoongCollectorMonitor::mHostname + LoongCollectorMonitor::mIpAddr + + ILOGTAIL_VERSION + GetProcessExecutionDir()); // Collect status information to send. LogGroup logGroup; logGroup.set_category(category); - logGroup.set_source(LogFileProfiler::mIpAddr); + logGroup.set_source(LoongCollectorMonitor::mIpAddr); Log* logPtr = logGroup.add_logs(); SetLogTime(logPtr, AppConfig::GetInstance()->EnableLogTimeAutoAdjust() ? now.tv_sec + GetTimeDelta() : now.tv_sec); // CPU usage of Logtail process. @@ -278,11 +287,11 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { AddLogContent(logPtr, "instance_id", Application::GetInstance()->GetInstanceId()); AddLogContent(logPtr, "instance_key", id); // Host informations. - AddLogContent(logPtr, "ip", LogFileProfiler::mIpAddr); - AddLogContent(logPtr, "hostname", LogFileProfiler::mHostname); + AddLogContent(logPtr, "ip", LoongCollectorMonitor::mIpAddr); + AddLogContent(logPtr, "hostname", LoongCollectorMonitor::mHostname); AddLogContent(logPtr, "os", OS_NAME); - AddLogContent(logPtr, "os_detail", LogFileProfiler::mOsDetail); - AddLogContent(logPtr, "user", LogFileProfiler::mUsername); + AddLogContent(logPtr, "os_detail", LoongCollectorMonitor::mOsDetail); + AddLogContent(logPtr, "user", LoongCollectorMonitor::mUsername); #if defined(__linux__) AddLogContent(logPtr, "load", GetLoadAvg()); #endif @@ -310,9 +319,9 @@ bool LogtailMonitor::SendStatusProfile(bool suicide) { AddLogContent(logPtr, "metric_json", MetricToString()); AddLogContent(logPtr, "status", CheckLogtailStatus()); - AddLogContent(logPtr, "ecs_instance_id", LogFileProfiler::mECSInstanceID); - AddLogContent(logPtr, "ecs_user_id", LogFileProfiler::mECSUserID); - AddLogContent(logPtr, "ecs_regioon_id", LogFileProfiler::mECSRegionID); + AddLogContent(logPtr, "ecs_instance_id", LoongCollectorMonitor::mECSInstanceID); + AddLogContent(logPtr, "ecs_user_id", LoongCollectorMonitor::mECSUserID); + AddLogContent(logPtr, "ecs_regioon_id", LoongCollectorMonitor::mECSRegionID); ClearMetric(); if (!mIsThreadRunning) @@ -500,10 +509,10 @@ bool LogtailMonitor::IsHostIpChanged() { if (ip.empty()) { ip = GetAnyAvailableIP(); } - if (ip != LogFileProfiler::mIpAddr) { + if (ip != LoongCollectorMonitor::mIpAddr) { LOG_ERROR(sLogger, ("error", "host ip changed during running, prepare to restart Logtail")( - "original ip", LogFileProfiler::mIpAddr)("current ip", ip)); + "original ip", LoongCollectorMonitor::mIpAddr)("current ip", ip)); return true; } return false; @@ -686,17 +695,36 @@ bool LogtailMonitor::CalOsCpuStat() { #endif LoongCollectorMonitor* LoongCollectorMonitor::GetInstance() { - static LoongCollectorMonitor instance; - return &instance; + static LoongCollectorMonitor* instance = new LoongCollectorMonitor(); + return instance; +} + +LoongCollectorMonitor::LoongCollectorMonitor() { + mHostname = GetHostName(); +#if defined(_MSC_VER) + mHostname = EncodingConverter::GetInstance()->FromACPToUTF8(mHostname); +#endif + mIpAddr = GetHostIp(); + mOsDetail = GetOsDetail(); + mUsername = GetUsername(); + // TODO: This may take up to 3s to construct the object. This is bad. + ECSMeta ecsMeta = FetchECSMeta(); + mECSInstanceID = ecsMeta.instanceID; + mECSUserID = ecsMeta.userID; + mECSRegionID = ecsMeta.regionID; +} + +LoongCollectorMonitor::~LoongCollectorMonitor() { } void LoongCollectorMonitor::Init() { // create metric record MetricLabels labels; labels.emplace_back(METRIC_LABEL_KEY_INSTANCE_ID, Application::GetInstance()->GetInstanceId()); - labels.emplace_back(METRIC_LABEL_KEY_START_TIME, LogFileProfiler::mStartTime); + labels.emplace_back(METRIC_LABEL_KEY_START_TIME, mStartTime); + labels.emplace_back(METRIC_LABEL_KEY_HOSTNAME, mHostname); labels.emplace_back(METRIC_LABEL_KEY_OS, OS_NAME); - labels.emplace_back(METRIC_LABEL_KEY_OS_DETAIL, LogFileProfiler::mOsDetail); + labels.emplace_back(METRIC_LABEL_KEY_OS_DETAIL, mOsDetail); labels.emplace_back(METRIC_LABEL_KEY_UUID, Application::GetInstance()->GetUUID()); labels.emplace_back(METRIC_LABEL_KEY_VERSION, ILOGTAIL_VERSION); DynamicMetricLabels dynamicLabels; diff --git a/core/monitor/Monitor.h b/core/monitor/Monitor.h index 4705236d32..c66a47218e 100644 --- a/core/monitor/Monitor.h +++ b/core/monitor/Monitor.h @@ -21,8 +21,8 @@ #include #include -#include "MetricManager.h" #include "MetricConstants.h" +#include "MetricManager.h" #include "MetricStore.h" #if defined(_MSC_VER) @@ -199,7 +199,20 @@ class LoongCollectorMonitor { void SetAgentOpenFdTotal(uint64_t total) { mAgentOpenFdTotal->Set(total); } void SetAgentConfigTotal(uint64_t total) { mAgentConfigTotal->Set(total); } + static std::string mHostname; + static std::string mIpAddr; + static std::string mOsDetail; + static std::string mUsername; + static int32_t mSystemBootTime; + static std::string mECSInstanceID; + static std::string mECSUserID; + static std::string mECSRegionID; + static std::string mStartTime; + private: + LoongCollectorMonitor(); + ~LoongCollectorMonitor(); + // MetricRecord MetricsRecordRef mMetricsRecordRef; diff --git a/core/monitor/metric_constants/AgentMetrics.cpp b/core/monitor/metric_constants/AgentMetrics.cpp index ad17130e6c..47a6866292 100644 --- a/core/monitor/metric_constants/AgentMetrics.cpp +++ b/core/monitor/metric_constants/AgentMetrics.cpp @@ -22,6 +22,7 @@ namespace logtail { const string METRIC_LABEL_KEY_ALIUIDS = "aliuids"; const string METRIC_LABEL_KEY_INSTANCE_ID = "instance_id"; const string METRIC_LABEL_KEY_START_TIME = "start_time"; +const string METRIC_LABEL_KEY_HOSTNAME = "hostname"; const string METRIC_LABEL_KEY_OS = "os"; const string METRIC_LABEL_KEY_OS_DETAIL = "os_detail"; const string METRIC_LABEL_KEY_PROJECT = "project"; diff --git a/core/monitor/metric_constants/MetricConstants.h b/core/monitor/metric_constants/MetricConstants.h index 73a62d30bf..95a6eb1af2 100644 --- a/core/monitor/metric_constants/MetricConstants.h +++ b/core/monitor/metric_constants/MetricConstants.h @@ -26,6 +26,7 @@ namespace logtail { extern const std::string METRIC_LABEL_KEY_ALIUIDS; extern const std::string METRIC_LABEL_KEY_INSTANCE_ID; extern const std::string METRIC_LABEL_KEY_START_TIME; +extern const std::string METRIC_LABEL_KEY_HOSTNAME; extern const std::string METRIC_LABEL_KEY_OS; extern const std::string METRIC_LABEL_KEY_OS_DETAIL; extern const std::string METRIC_LABEL_KEY_PROJECT; diff --git a/core/pipeline/PipelineContext.h b/core/pipeline/PipelineContext.h index 7bbafa7b0a..2c4a7f580d 100644 --- a/core/pipeline/PipelineContext.h +++ b/core/pipeline/PipelineContext.h @@ -23,7 +23,6 @@ #include "logger/Logger.h" #include "models/PipelineEventGroup.h" -#include "monitor/LogFileProfiler.h" #include "monitor/AlarmManager.h" #include "pipeline/GlobalConfig.h" #include "pipeline/queue/QueueKey.h" @@ -83,7 +82,7 @@ class PipelineContext { void SetIsFirstProcessorApsaraFlag(bool flag) { mIsFirstProcessorApsara = flag; } bool IsFirstProcessorJson() const { return mIsFirstProcessorJson; } void SetIsFirstProcessorJsonFlag(bool flag) { mIsFirstProcessorJson = flag; } - bool IsExactlyOnceEnabled() const {return mEnableExactlyOnce; } + bool IsExactlyOnceEnabled() const { return mEnableExactlyOnce; } void SetExactlyOnceFlag(bool flag) { mEnableExactlyOnce = flag; } bool HasNativeProcessors() const { return mHasNativeProcessors; } void SetHasNativeProcessorsFlag(bool flag) { mHasNativeProcessors = flag; } @@ -91,7 +90,6 @@ class PipelineContext { void SetIsFlushingThroughGoPipelineFlag(bool flag) { mIsFlushingThroughGoPipeline = flag; } ProcessProfile& GetProcessProfile() const { return mProcessProfile; } - // LogFileProfiler& GetProfiler() { return *mProfiler; } const Logger::logger& GetLogger() const { return mLogger; } AlarmManager& GetAlarm() const { return *mAlarm; }; @@ -114,7 +112,6 @@ class PipelineContext { bool mIsFlushingThroughGoPipeline = false; mutable ProcessProfile mProcessProfile; - // LogFileProfiler* mProfiler = LogFileProfiler::GetInstance(); Logger::logger mLogger = sLogger; AlarmManager* mAlarm = AlarmManager::GetInstance(); }; diff --git a/core/plugin/flusher/sls/SLSClientManager.cpp b/core/plugin/flusher/sls/SLSClientManager.cpp index 33a0381bef..9221aad91b 100644 --- a/core/plugin/flusher/sls/SLSClientManager.cpp +++ b/core/plugin/flusher/sls/SLSClientManager.cpp @@ -21,7 +21,7 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" +#include "monitor/Monitor.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "plugin/flusher/sls/SendResult.h" #include "sdk/Exception.h" @@ -122,7 +122,7 @@ void SLSClientManager::Init() { STRING_FLAG(default_access_key_id), STRING_FLAG(default_access_key), INT32_FLAG(sls_client_send_timeout), - LogFileProfiler::mIpAddr, + LoongCollectorMonitor::mIpAddr, AppConfig::GetInstance()->GetBindInterface())); SLSControl::GetInstance()->SetSlsSendClientCommonParam(mProbeNetworkClient.get()); mProbeNetworkThreadRes = async(launch::async, &SLSClientManager::ProbeNetworkThread, this); @@ -132,7 +132,7 @@ void SLSClientManager::Init() { STRING_FLAG(default_access_key_id), STRING_FLAG(default_access_key), INT32_FLAG(sls_client_send_timeout), - LogFileProfiler::mIpAddr, + LoongCollectorMonitor::mIpAddr, AppConfig::GetInstance()->GetBindInterface())); SLSControl::GetInstance()->SetSlsSendClientCommonParam(mUpdateRealIpClient.get()); mUpdateRealIpThreadRes = async(launch::async, &SLSClientManager::UpdateRealIpThread, this); @@ -232,7 +232,7 @@ sdk::Client* SLSClientManager::GetClient(const string& region, const string& ali "", "", INT32_FLAG(sls_client_send_timeout), - LogFileProfiler::mIpAddr, + LoongCollectorMonitor::mIpAddr, AppConfig::GetInstance()->GetBindInterface()); SLSControl::GetInstance()->SetSlsSendClientCommonParam(client.get()); ResetClientPort(region, client.get()); diff --git a/core/plugin/processor/inner/ProcessorTagNative.cpp b/core/plugin/processor/inner/ProcessorTagNative.cpp index e9f9926339..dbabe3fc34 100644 --- a/core/plugin/processor/inner/ProcessorTagNative.cpp +++ b/core/plugin/processor/inner/ProcessorTagNative.cpp @@ -21,8 +21,9 @@ #include "app_config/AppConfig.h" #include "application/Application.h" #include "common/Flags.h" -#include "protobuf/sls/sls_logs.pb.h" +#include "monitor/Monitor.h" #include "pipeline/Pipeline.h" +#include "protobuf/sls/sls_logs.pb.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #endif @@ -69,8 +70,8 @@ void ProcessorTagNative::Process(PipelineEventGroup& logGroup) { } // process level - logGroup.SetTagNoCopy(LOG_RESERVED_KEY_HOSTNAME, LogFileProfiler::mHostname); - logGroup.SetTagNoCopy(LOG_RESERVED_KEY_SOURCE, LogFileProfiler::mIpAddr); + logGroup.SetTagNoCopy(LOG_RESERVED_KEY_HOSTNAME, LoongCollectorMonitor::mHostname); + logGroup.SetTagNoCopy(LOG_RESERVED_KEY_SOURCE, LoongCollectorMonitor::mIpAddr); auto sb = logGroup.GetSourceBuffer()->CopyString(Application::GetInstance()->GetUUID()); logGroup.SetTagNoCopy(LOG_RESERVED_KEY_MACHINE_UUID, StringView(sb.data, sb.size)); static const vector& sEnvTags = AppConfig::GetInstance()->GetEnvTags(); diff --git a/core/runner/ProcessorRunner.cpp b/core/runner/ProcessorRunner.cpp index 37e41da27b..15a213919f 100644 --- a/core/runner/ProcessorRunner.cpp +++ b/core/runner/ProcessorRunner.cpp @@ -19,7 +19,6 @@ #include "common/Flags.h" #include "go_pipeline/LogtailPlugin.h" #include "models/EventPool.h" -#include "monitor/LogFileProfiler.h" #include "monitor/AlarmManager.h" #include "monitor/metric_constants/MetricConstants.h" #include "pipeline/PipelineManager.h" diff --git a/core/sls_control/SLSControl.cpp b/core/sls_control/SLSControl.cpp index ff15b99fe8..edc43ee76c 100644 --- a/core/sls_control/SLSControl.cpp +++ b/core/sls_control/SLSControl.cpp @@ -20,13 +20,12 @@ #include #endif -#include "curl/curl.h" - #include "app_config/AppConfig.h" #include "common/Flags.h" #include "common/version.h" +#include "curl/curl.h" #include "logger/Logger.h" -#include "monitor/LogFileProfiler.h" +#include "monitor/Monitor.h" #ifdef __ENTERPRISE__ #include "sls_control/EnterpriseSLSControl.h" #endif @@ -78,8 +77,8 @@ void SLSControl::GenerateUserAgent() { if (-1 == uname(buf)) { LOG_WARNING( sLogger, - ("get os info part of user agent failed", errno)("use default os info", LogFileProfiler::mOsDetail)); - os = LogFileProfiler::mOsDetail; + ("get os info part of user agent failed", errno)("use default os info", LoongCollectorMonitor::mOsDetail)); + os = LoongCollectorMonitor::mOsDetail; } else { char* pch = strchr(buf->release, '-'); if (pch) { @@ -93,10 +92,10 @@ void SLSControl::GenerateUserAgent() { } delete buf; #elif defined(_MSC_VER) - os = LogFileProfiler::mOsDetail; + os = LoongCollectorMonitor::mOsDetail; #endif - mUserAgent = string("ilogtail/") + ILOGTAIL_VERSION + " (" + os + ") ip/" + LogFileProfiler::mIpAddr + " env/" + mUserAgent = string("ilogtail/") + ILOGTAIL_VERSION + " (" + os + ") ip/" + LoongCollectorMonitor::mIpAddr + " env/" + GetRunningEnvironment(); if (!STRING_FLAG(custom_user_agent).empty()) { mUserAgent += " " + STRING_FLAG(custom_user_agent); diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index fe41260944..276c6a14ef 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -24,6 +24,7 @@ #include "config/watcher/ConfigWatcher.h" #include "config/watcher/InstanceConfigWatcher.h" #include "gmock/gmock.h" +#include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" #include "unittest/Unittest.h" @@ -296,8 +297,8 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { APSARA_TEST_TRUE(heartbeatReq.capabilities() & configserver::proto::v2::AcceptsPipelineConfig); APSARA_TEST_EQUAL(heartbeatReq.instance_id(), provider.GetInstanceId()); APSARA_TEST_EQUAL(heartbeatReq.agent_type(), "LoongCollector"); - APSARA_TEST_EQUAL(heartbeatReq.attributes().ip(), LogFileProfiler::mIpAddr); - APSARA_TEST_EQUAL(heartbeatReq.attributes().hostname(), LogFileProfiler::mHostname); + APSARA_TEST_EQUAL(heartbeatReq.attributes().ip(), LoongCollectorMonitor::mIpAddr); + APSARA_TEST_EQUAL(heartbeatReq.attributes().hostname(), LoongCollectorMonitor::mHostname); APSARA_TEST_EQUAL(heartbeatReq.attributes().version(), ILOGTAIL_VERSION); auto it = heartbeatReq.tags().begin(); APSARA_TEST_EQUAL(it->name(), "key1"); @@ -511,8 +512,8 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { APSARA_TEST_TRUE(heartbeatReq.capabilities() & configserver::proto::v2::AcceptsPipelineConfig); APSARA_TEST_EQUAL(heartbeatReq.instance_id(), provider.GetInstanceId()); APSARA_TEST_EQUAL(heartbeatReq.agent_type(), "LoongCollector"); - APSARA_TEST_EQUAL(heartbeatReq.attributes().ip(), LogFileProfiler::mIpAddr); - APSARA_TEST_EQUAL(heartbeatReq.attributes().hostname(), LogFileProfiler::mHostname); + APSARA_TEST_EQUAL(heartbeatReq.attributes().ip(), LoongCollectorMonitor::mIpAddr); + APSARA_TEST_EQUAL(heartbeatReq.attributes().hostname(), LoongCollectorMonitor::mHostname); APSARA_TEST_EQUAL(heartbeatReq.attributes().version(), ILOGTAIL_VERSION); auto it = heartbeatReq.tags().begin(); APSARA_TEST_EQUAL(it->name(), "key1"); diff --git a/core/unittest/config/ConfigUpdatorUnittest.cpp b/core/unittest/config/ConfigUpdatorUnittest.cpp index 13e4846bd3..1ef9a6825f 100644 --- a/core/unittest/config/ConfigUpdatorUnittest.cpp +++ b/core/unittest/config/ConfigUpdatorUnittest.cpp @@ -17,29 +17,31 @@ #include #endif #include -#include +#include #include -#include -#include + +#include #include -#include -#include "file_server/event_handler/EventHandler.h" -#include "file_server/ConfigManager.h" -#include "file_server/reader/LogFileReader.h" +#include +#include + +#include "AlarmManager.h" #include "AppConfig.h" -#include "Monitor.h" -#include "EventDispatcher.h" #include "CheckPointManager.h" -#include "file_server/event_handler/LogInput.h" +#include "EventDispatcher.h" +#include "Monitor.h" #include "Sender.h" -#include "sls_logs.pb.h" -#include "AlarmManager.h" +#include "common/FileSystemUtil.h" #include "common/Flags.h" #include "common/Lock.h" #include "constants/Constants.h" -#include "common/FileSystemUtil.h" +#include "file_server/ConfigManager.h" +#include "file_server/event_handler/EventHandler.h" +#include "file_server/event_handler/LogInput.h" +#include "file_server/reader/LogFileReader.h" #include "logger/Logger.h" #include "sdk/Common.h" +#include "sls_logs.pb.h" using namespace std; using namespace logtail::sdk; @@ -1704,7 +1706,7 @@ void ConfigUpdatorUnittest::GetLogContent(LogType logType, char* buffer) { if (logType == REGEX_LOG) { strftime(timeBuffer, sizeof(timeBuffer), " - - [%d/%b/%Y:%R:%S +0800] ", &timeInfo); - sprintf(buffer, "%s%s%s", LogFileProfiler::GetInstance()->mIpAddr.c_str(), timeBuffer, buffer2); + sprintf(buffer, "%s%s%s", LoongCollectorMonitor::GetInstance()->mIpAddr.c_str(), timeBuffer, buffer2); if ((rand()) % 2 == 0) strcat(buffer, buffer3); strcat(buffer, "\n"); @@ -2637,16 +2639,16 @@ void ConfigUpdatorUnittest::TestValidWildcardPath2() { void ConfigUpdatorUnittest::TestWithinMaxDepth() { // No wildcard. - PipelineConfig* cfg_1 - = new PipelineConfig(PS + "abc" + PS + "de" + PS + "f", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 0, "cat"); + PipelineConfig* cfg_1 = new PipelineConfig( + PS + "abc" + PS + "de" + PS + "f", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 0, "cat"); EXPECT_EQ(cfg_1->WithinMaxDepth(PS + "abc"), false); EXPECT_EQ(cfg_1->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f"), true); EXPECT_EQ(cfg_1->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "fx"), false); EXPECT_EQ(cfg_1->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f" + PS + "ghi"), false); delete cfg_1; // To be compatible with old settings - PipelineConfig* cfg_2 - = new PipelineConfig(PS + "abc" + PS + "de" + PS + "f", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, -1, "cat"); + PipelineConfig* cfg_2 = new PipelineConfig( + PS + "abc" + PS + "de" + PS + "f", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, -1, "cat"); EXPECT_EQ(cfg_2->WithinMaxDepth(PS + "abc"), true); EXPECT_EQ(cfg_2->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f"), true); EXPECT_EQ(cfg_2->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "fx"), true); @@ -2654,8 +2656,8 @@ void ConfigUpdatorUnittest::TestWithinMaxDepth() { EXPECT_EQ(cfg_2->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f" + PS + "ghi" + PS + "agec" + PS + "egegt"), true); delete cfg_2; - PipelineConfig* cfg_3 - = new PipelineConfig(PS + "abc" + PS + "de" + PS + "f", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 3, "cat"); + PipelineConfig* cfg_3 = new PipelineConfig( + PS + "abc" + PS + "de" + PS + "f", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 3, "cat"); EXPECT_EQ(cfg_3->WithinMaxDepth(PS + "abc"), false); EXPECT_EQ(cfg_3->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f"), true); EXPECT_EQ(cfg_3->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "fx"), false); @@ -2671,16 +2673,27 @@ void ConfigUpdatorUnittest::TestWithinMaxDepth() { delete cfg_3; // Wildcard. - PipelineConfig* cfg_4 - = new PipelineConfig(PS + "ab?" + PS + "de" + PS + "*", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 0, "cat"); + PipelineConfig* cfg_4 = new PipelineConfig( + PS + "ab?" + PS + "de" + PS + "*", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 0, "cat"); EXPECT_EQ(cfg_4->WithinMaxDepth(PS + "abc"), false); EXPECT_EQ(cfg_4->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f"), true); EXPECT_EQ(cfg_4->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "xyz"), true); EXPECT_EQ(cfg_4->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f" + PS + "ghi"), false); delete cfg_4; // To be compatible with old settings. - PipelineConfig* cfg_5 = new PipelineConfig( - PS + "abc" + PS + "de?" + PS + "f*" + PS + "xyz", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, -1, "cat", ""); + PipelineConfig* cfg_5 = new PipelineConfig(PS + "abc" + PS + "de?" + PS + "f*" + PS + "xyz", + "x.log", + REGEX_LOG, + "a", + "", + "", + "", + "prj", + true, + 0, + -1, + "cat", + ""); EXPECT_EQ(cfg_5->WithinMaxDepth(PS + "abc"), true); EXPECT_EQ(cfg_5->WithinMaxDepth(PS + "abc" + PS + "def" + PS + "fgz"), true); EXPECT_EQ(cfg_5->WithinMaxDepth(PS + "abc" + PS + "def" + PS + "fgz" + PS + "xyz0"), true); @@ -2691,8 +2704,8 @@ void ConfigUpdatorUnittest::TestWithinMaxDepth() { true); delete cfg_5; - PipelineConfig* cfg_6 - = new PipelineConfig(PS + "abc" + PS + "d?" + PS + "f*", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 3, "cat"); + PipelineConfig* cfg_6 = new PipelineConfig( + PS + "abc" + PS + "d?" + PS + "f*", "x.log", REGEX_LOG, "a", "", "", "", "prj", true, 0, 3, "cat"); EXPECT_EQ(cfg_6->WithinMaxDepth(PS + "abc"), false); EXPECT_EQ(cfg_6->WithinMaxDepth(PS + "abc" + PS + "de"), false); EXPECT_EQ(cfg_6->WithinMaxDepth(PS + "abc" + PS + "de" + PS + "f"), true); diff --git a/core/unittest/processor/ProcessorTagNativeUnittest.cpp b/core/unittest/processor/ProcessorTagNativeUnittest.cpp index 3e784f935e..d634c215c8 100644 --- a/core/unittest/processor/ProcessorTagNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorTagNativeUnittest.cpp @@ -14,9 +14,10 @@ #include -#include "constants/Constants.h" #include "config/PipelineConfig.h" +#include "constants/Constants.h" #include "file_server/ConfigManager.h" +#include "monitor/Monitor.h" #include "pipeline/Pipeline.h" #include "plugin/processor/inner/ProcessorTagNative.h" #include "unittest/Unittest.h" @@ -34,7 +35,7 @@ class ProcessorTagNativeUnittest : public ::testing::Test { protected: void SetUp() override { mContext.SetConfigName("project##config_0"); - LogFileProfiler::GetInstance(); + LoongCollectorMonitor::GetInstance(); #ifdef __ENTERPRISE__ EnterpriseConfigProvider::GetInstance()->SetUserDefinedIdSet(std::vector{"machine_group"}); #endif @@ -102,7 +103,7 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_EQUAL_FATAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_PATH), eventGroup.GetTag(LOG_RESERVED_KEY_PATH)); APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_HOSTNAME)); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mHostname, eventGroup.GetTag(LOG_RESERVED_KEY_HOSTNAME)); + APSARA_TEST_EQUAL_FATAL(LoongCollectorMonitor::mHostname, eventGroup.GetTag(LOG_RESERVED_KEY_HOSTNAME)); #ifdef __ENTERPRISE__ APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_USER_DEFINED_ID)); APSARA_TEST_EQUAL_FATAL(EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(), diff --git a/core/unittest/sender/SenderUnittest.cpp b/core/unittest/sender/SenderUnittest.cpp index e2ac680333..0f01300bb2 100644 --- a/core/unittest/sender/SenderUnittest.cpp +++ b/core/unittest/sender/SenderUnittest.cpp @@ -46,7 +46,6 @@ #include #include "checkpoint/CheckpointManagerV2.h" -#include "constants/Constants.h" #include "common/FileEncryption.h" #include "common/FileSystemUtil.h" #include "common/Lock.h" @@ -55,10 +54,11 @@ #include "common/StringTools.h" #include "common/Thread.h" #include "common/WaitObject.h" +#include "constants/Constants.h" #include "file_server/event_handler/LogInput.h" #include "logger/Logger.h" -#include "monitor/LogIntegrity.h" #include "monitor/AlarmManager.h" +#include "monitor/LogIntegrity.h" #include "protobuf/sls/metric.pb.h" #include "protobuf/sls/sls_logs.pb.h" #include "runner/ProcessorRunner.h" @@ -850,7 +850,9 @@ class SenderUnittest : public ::testing::Test { sLogger->set_level(spdlog::level::trace); printf("Test case setup.\n"); srand(time(NULL)); - Sender::Instance()->AddEndpointEntry(STRING_FLAG(default_region_name), STRING_FLAG(logtail_send_address), SLSClientManager::EndpointSourceType::LOCAL); + Sender::Instance()->AddEndpointEntry(STRING_FLAG(default_region_name), + STRING_FLAG(logtail_send_address), + SLSClientManager::EndpointSourceType::LOCAL); STRING_FLAG(profile_project_name) = "sls-admin"; INT32_FLAG(sls_host_update_interval) = 1; INT32_FLAG(logtail_alarm_interval) = 600; @@ -2246,59 +2248,6 @@ class SenderUnittest : public ::testing::Test { LOG_INFO(sLogger, ("TestFlushOut() end", time(NULL))); } - void TestDumpSnapshot() { - LOG_INFO(sLogger, ("TestDumpSnapshot() begin", time(NULL))); - CaseSetUp(); - EnableNetWork(); - LogFileProfiler::GetInstance()->mSendInterval = 3600; - LogFileProfiler::GetInstance()->SendProfileData(true); // Update mLastSendTime. - - OneJob(10, gRootDir, "Job", true, time(NULL)); - WaitForFileBeenRead(); - sleep(5); - LogFileProfiler::GetInstance()->SendProfileData(true); - Json::Value root; - ParseConfig(STRING_FLAG(logtail_profile_snapshot), root); - LOG_INFO(sLogger, ("snapshot", root.toStyledString())); - APSARA_TEST_TRUE(root["version"].asString().size() > 1); - APSARA_TEST_EQUAL(root["ip"].asString(), LogFileProfiler::mIpAddr); - int32_t timeInterval = root["end_time"].asInt64() - root["begin_time"].asInt64(); -#if defined(__linux__) - APSARA_TEST_TRUE_DESC(timeInterval >= 5 && timeInterval <= 7, timeInterval); -#elif defined(_MSC_VER) - EXPECT_TRUE(timeInterval >= 5 && timeInterval <= 20); -#endif - APSARA_TEST_EQUAL(root["detail"].size(), 1); - Json::Value categoryDetail = root["detail"][0]; - APSARA_TEST_EQUAL(categoryDetail["project"].asString(), "1000000_proj"); - APSARA_TEST_EQUAL(categoryDetail["logstore"].asString(), "app_log"); - APSARA_TEST_EQUAL(StringTo(categoryDetail["split_lines"].asString()), 10); - APSARA_TEST_EQUAL(StringTo(categoryDetail["parse_fail_lines"].asString()), 0); - APSARA_TEST_TRUE(StringTo(categoryDetail["read_bytes"].asString()) > 0); - - OneJob(100, gRootDir, "Job", true, time(NULL)); - WaitForFileBeenRead(); - sleep(5); - LogFileProfiler::GetInstance()->SendProfileData(true); - - ParseConfig(STRING_FLAG(logtail_profile_snapshot), root); - LOG_INFO(sLogger, ("snapshot", root.toStyledString())); - APSARA_TEST_TRUE(root["version"].asString().size() > 1); - APSARA_TEST_EQUAL(root["ip"].asString(), LogFileProfiler::mIpAddr); - timeInterval = root["end_time"].asInt64() - root["begin_time"].asInt64(); - APSARA_TEST_TRUE_DESC(timeInterval >= 5 && timeInterval <= 20, timeInterval); - APSARA_TEST_EQUAL(root["detail"].size(), 1); - categoryDetail = root["detail"][0]; - APSARA_TEST_EQUAL(categoryDetail["project"].asString(), "1000000_proj"); - APSARA_TEST_EQUAL(categoryDetail["logstore"].asString(), "app_log"); - APSARA_TEST_EQUAL(StringTo(categoryDetail["split_lines"].asString()), 100); - APSARA_TEST_EQUAL(StringTo(categoryDetail["parse_fail_lines"].asString()), 0); - APSARA_TEST_TRUE(StringTo(categoryDetail["read_bytes"].asString()) > 0); - - CaseCleanUp(); - LOG_INFO(sLogger, ("TestDumpSnapshot() end", time(NULL))); - } - void TestMergeByMinute() { LOG_INFO(sLogger, ("TestMergeByMinute() begin", time(NULL))); CaseSetUp(); From ac991b84605c4a01571eb4cad0f001e50b38ea39 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Tue, 19 Nov 2024 16:58:03 +0800 Subject: [PATCH 02/11] add support fot task pipeline (#1883) --- core/CMakeLists.txt | 1 + core/app_config/AppConfig.cpp | 1 + core/application/Application.cpp | 14 +- core/common/ParamExtractor.h | 39 ++ core/config/ConfigDiff.h | 20 +- core/config/ConfigUtil.cpp | 87 ++++ core/config/ConfigUtil.h | 36 ++ core/config/InstanceConfig.cpp | 30 -- core/config/PipelineConfig.cpp | 60 --- core/config/PipelineConfig.h | 8 - core/config/TaskConfig.cpp | 52 +++ core/config/TaskConfig.h | 46 ++ .../common_provider/CommonConfigProvider.cpp | 1 + core/config/provider/ConfigProvider.cpp | 6 +- core/config/watcher/ConfigWatcher.cpp | 180 +------- core/config/watcher/ConfigWatcher.h | 30 +- core/config/watcher/InstanceConfigWatcher.cpp | 21 +- core/config/watcher/InstanceConfigWatcher.h | 25 +- core/config/watcher/PipelineConfigWatcher.cpp | 315 +++++++++++++ core/config/watcher/PipelineConfigWatcher.h | 60 +++ core/task_pipeline/Task.h | 42 ++ core/task_pipeline/TaskPipeline.cpp | 35 ++ core/task_pipeline/TaskPipeline.h | 53 +++ core/task_pipeline/TaskPipelineManager.cpp | 106 +++++ core/task_pipeline/TaskPipelineManager.h | 62 +++ core/task_pipeline/TaskRegistry.cpp | 45 ++ core/task_pipeline/TaskRegistry.h | 57 +++ core/unittest/CMakeLists.txt | 1 + core/unittest/config/CMakeLists.txt | 10 +- .../config/CommonConfigProviderUnittest.cpp | 38 +- core/unittest/config/ConfigUpdateUnittest.cpp | 415 +++++++++++++----- .../unittest/config/ConfigWatcherUnittest.cpp | 37 +- core/unittest/config/TaskConfigUnittest.cpp | 151 +++++++ core/unittest/plugin/PluginMock.h | 25 ++ core/unittest/task_pipeline/CMakeLists.txt | 30 ++ .../TaskPipelineManagerUnittest.cpp | 40 ++ .../task_pipeline/TaskPipelineUnittest.cpp | 127 ++++++ .../task_pipeline/TaskRegistryUnittest.cpp | 50 +++ 38 files changed, 1868 insertions(+), 488 deletions(-) create mode 100644 core/config/ConfigUtil.cpp create mode 100644 core/config/ConfigUtil.h delete mode 100644 core/config/InstanceConfig.cpp create mode 100644 core/config/TaskConfig.cpp create mode 100644 core/config/TaskConfig.h create mode 100644 core/config/watcher/PipelineConfigWatcher.cpp create mode 100644 core/config/watcher/PipelineConfigWatcher.h create mode 100644 core/task_pipeline/Task.h create mode 100644 core/task_pipeline/TaskPipeline.cpp create mode 100644 core/task_pipeline/TaskPipeline.h create mode 100644 core/task_pipeline/TaskPipelineManager.cpp create mode 100644 core/task_pipeline/TaskPipelineManager.h create mode 100644 core/task_pipeline/TaskRegistry.cpp create mode 100644 core/task_pipeline/TaskRegistry.h create mode 100644 core/unittest/config/TaskConfigUnittest.cpp create mode 100644 core/unittest/task_pipeline/CMakeLists.txt create mode 100644 core/unittest/task_pipeline/TaskPipelineManagerUnittest.cpp create mode 100644 core/unittest/task_pipeline/TaskPipelineUnittest.cpp create mode 100644 core/unittest/task_pipeline/TaskRegistryUnittest.cpp diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index fe5810d906..20ea817dab 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -116,6 +116,7 @@ set(SUB_DIRECTORIES_LIST application app_config checkpoint container_manager metadata logger go_pipeline monitor monitor/metric_constants monitor/profile_sender models config config/watcher constants pipeline pipeline/batch pipeline/limiter pipeline/plugin pipeline/plugin/creator pipeline/plugin/instance pipeline/plugin/interface pipeline/queue pipeline/route pipeline/serializer + task_pipeline runner runner/sink/http protobuf/sls protobuf/models file_server file_server/event file_server/event_handler file_server/event_listener file_server/reader file_server/polling diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index c154ce0c9b..976a7b3339 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -25,6 +25,7 @@ #include "common/FileSystemUtil.h" #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" +#include "config/InstanceConfigManager.h" #include "config/watcher/InstanceConfigWatcher.h" #include "file_server/ConfigManager.h" #include "file_server/reader/LogFileReader.h" diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 6cef734be0..08df561cd2 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -32,7 +32,7 @@ #include "common/version.h" #include "config/ConfigDiff.h" #include "config/InstanceConfigManager.h" -#include "config/watcher/ConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" #include "config/watcher/InstanceConfigWatcher.h" #include "file_server/ConfigManager.h" #include "file_server/EventDispatcher.h" @@ -51,6 +51,7 @@ #include "runner/FlusherRunner.h" #include "runner/ProcessorRunner.h" #include "runner/sink/http/HttpSink.h" +#include "task_pipeline/TaskPipelineManager.h" #ifdef __ENTERPRISE__ #include "config/provider/EnterpriseConfigProvider.h" #include "config/provider/LegacyConfigProvider.h" @@ -218,7 +219,7 @@ void Application::Start() { // GCOVR_EXCL_START ("failed to create dir for local pipeline_config", "manual creation may be required")("error code", ec.value())("error msg", ec.message())); } - ConfigWatcher::GetInstance()->AddSource(localConfigPath.string()); + PipelineConfigWatcher::GetInstance()->AddSource(localConfigPath.string()); } #ifdef __ENTERPRISE__ @@ -275,9 +276,12 @@ void Application::Start() { // GCOVR_EXCL_START lastCheckTagsTime = curTime; } if (curTime - lastConfigCheckTime >= INT32_FLAG(config_scan_interval)) { - PipelineConfigDiff pipelineConfigDiff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - if (!pipelineConfigDiff.IsEmpty()) { - PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff); + auto configDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + if (!configDiff.first.IsEmpty()) { + PipelineManager::GetInstance()->UpdatePipelines(configDiff.first); + } + if (!configDiff.second.IsEmpty()) { + TaskPipelineManager::GetInstance()->UpdatePipelines(configDiff.second); } InstanceConfigDiff instanceConfigDiff = InstanceConfigWatcher::GetInstance()->CheckConfigDiff(); if (!instanceConfigDiff.IsEmpty()) { diff --git a/core/common/ParamExtractor.h b/core/common/ParamExtractor.h index 94432281ed..d63e8a1895 100644 --- a/core/common/ParamExtractor.h +++ b/core/common/ParamExtractor.h @@ -81,6 +81,45 @@ region); \ } +#define TASK_PARAM_ERROR_RETURN(logger, alarm, msg, module, config) \ + if (module.empty()) { \ + LOG_ERROR(logger, ("failed to parse config", msg)("config", config)); \ + alarm.SendAlarm(CATEGORY_CONFIG_ALARM, std::string(msg) + ": abort, config: " + config); \ + } else { \ + LOG_ERROR(logger, ("failed to parse config", msg)("module", module)("config", config)); \ + alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \ + std::string(msg) + ": abort, module: " + module + ", config: " + config); \ + } \ + return false +#define TASK_PARAM_WARNING_IGNORE(logger, alarm, msg, module, config) \ + if (module.empty()) { \ + LOG_WARNING(logger, \ + ("problem encountered in config parsing", msg)("action", "ignore param")("config", config)); \ + alarm.SendAlarm(CATEGORY_CONFIG_ALARM, std::string(msg) + ": ignore param, config: " + config); \ + } else { \ + LOG_WARNING(logger, \ + ("problem encountered in config parsing", \ + msg)("action", "ignore param")("module", module)("config", config)); \ + alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \ + std::string(msg) + ": ignore param, module: " + module + ", config: " + config); \ + } +#define TASK_PARAM_WARNING_DEFAULT(logger, alarm, msg, val, module, config) \ + if (module.empty()) { \ + LOG_WARNING(logger, \ + ("problem encountered in config parsing", \ + msg)("action", "use default value instead")("default value", ToString(val))("config", config)); \ + alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \ + std::string(msg) + ": use default value instead, default value: " + ToString(val) \ + + ", config: " + config); \ + } else { \ + LOG_WARNING(logger, \ + ("problem encountered in config parsing", msg)("action", "use default value instead")( \ + "default value", ToString(val))("module", module)("config", config)); \ + alarm.SendAlarm(CATEGORY_CONFIG_ALARM, \ + std::string(msg) + ": use default value instead, default value: " + ToString(val) \ + + ", module: " + module + ", config: " + config); \ + } + namespace logtail { const std::string noModule = ""; diff --git a/core/config/ConfigDiff.h b/core/config/ConfigDiff.h index 8dc3ed2d26..38fa7ba00a 100644 --- a/core/config/ConfigDiff.h +++ b/core/config/ConfigDiff.h @@ -21,23 +21,21 @@ #include "config/InstanceConfig.h" #include "config/PipelineConfig.h" +#include "config/TaskConfig.h" namespace logtail { -class PipelineConfigDiff { -public: - std::vector mAdded; - std::vector mModified; +template +struct ConfigDiff { + std::vector mAdded; + std::vector mModified; std::vector mRemoved; - bool IsEmpty() { return mRemoved.empty() && mAdded.empty() && mModified.empty(); } -}; -class InstanceConfigDiff { -public: - std::vector mAdded; - std::vector mModified; - std::vector mRemoved; bool IsEmpty() { return mRemoved.empty() && mAdded.empty() && mModified.empty(); } }; +using PipelineConfigDiff = ConfigDiff; +using TaskConfigDiff = ConfigDiff; +using InstanceConfigDiff = ConfigDiff; + } // namespace logtail diff --git a/core/config/ConfigUtil.cpp b/core/config/ConfigUtil.cpp new file mode 100644 index 0000000000..2c3975f4c7 --- /dev/null +++ b/core/config/ConfigUtil.cpp @@ -0,0 +1,87 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "config/ConfigUtil.h" + +#include "common/FileSystemUtil.h" +#include "common/JsonUtil.h" +#include "common/YamlUtil.h" +#include "logger/Logger.h" + +using namespace std; + +namespace logtail { + +bool LoadConfigDetailFromFile(const filesystem::path& filepath, Json::Value& detail) { + const string& ext = filepath.extension().string(); + const string& configName = filepath.stem().string(); + if (configName == "region_config") { + return false; + } + if (ext != ".yaml" && ext != ".yml" && ext != ".json") { + LOG_WARNING(sLogger, ("unsupported config file format", "skip current object")("filepath", filepath)); + return false; + } + string content; + if (!ReadFile(filepath.string(), content)) { + LOG_WARNING(sLogger, ("failed to open config file", "skip current object")("filepath", filepath)); + return false; + } + if (content.empty()) { + LOG_WARNING(sLogger, ("empty config file", "skip current object")("filepath", filepath)); + return false; + } + string errorMsg; + if (!ParseConfigDetail(content, ext, detail, errorMsg)) { + LOG_WARNING(sLogger, + ("config file format error", "skip current object")("error msg", errorMsg)("filepath", filepath)); + return false; + } + return true; +} + +bool ParseConfigDetail(const string& content, const string& extension, Json::Value& detail, string& errorMsg) { + if (extension == ".json") { + return ParseJsonTable(content, detail, errorMsg); + } else if (extension == ".yaml" || extension == ".yml") { + YAML::Node yamlRoot; + if (!ParseYamlTable(content, yamlRoot, errorMsg)) { + return false; + } + detail = ConvertYamlToJson(yamlRoot); + return true; + } + return false; +} + +bool IsConfigEnabled(const string& name, const Json::Value& detail) { + const char* key = "enable"; + const Json::Value* itr = detail.find(key, key + strlen(key)); + if (itr != nullptr) { + if (!itr->isBool()) { + LOG_WARNING(sLogger, + ("problem encountered in config parsing", + "param enable is not of type bool")("action", "ignore the config")("config", name)); + return false; + } + return itr->asBool(); + } + return true; +} + +ConfigType GetConfigType(const Json::Value& detail) { + return detail.isMember("task") ? ConfigType::Task : ConfigType::Pipeline; +} + +} // namespace logtail diff --git a/core/config/ConfigUtil.h b/core/config/ConfigUtil.h new file mode 100644 index 0000000000..c5fd7d0ae2 --- /dev/null +++ b/core/config/ConfigUtil.h @@ -0,0 +1,36 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace logtail { + +enum class ConfigType { Pipeline, Task }; + +bool LoadConfigDetailFromFile(const std::filesystem::path& filepath, Json::Value& detail); +bool ParseConfigDetail(const std::string& content, + const std::string& extenstion, + Json::Value& detail, + std::string& errorMsg); +bool IsConfigEnabled(const std::string& name, const Json::Value& detail); +ConfigType GetConfigType(const Json::Value& detail); + +} // namespace logtail diff --git a/core/config/InstanceConfig.cpp b/core/config/InstanceConfig.cpp deleted file mode 100644 index 94c89ca684..0000000000 --- a/core/config/InstanceConfig.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2023 iLogtail Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "config/InstanceConfig.h" - -#include - -#include "app_config/AppConfig.h" -#include "common/FileSystemUtil.h" -#include "common/Flags.h" -#include "common/JsonUtil.h" -#include "common/ParamExtractor.h" -#include "common/YamlUtil.h" -#include "pipeline/plugin/PluginRegistry.h" - - -using namespace std; - -namespace logtail {} // namespace logtail diff --git a/core/config/PipelineConfig.cpp b/core/config/PipelineConfig.cpp index e80d3fae8b..33cf8ce6df 100644 --- a/core/config/PipelineConfig.cpp +++ b/core/config/PipelineConfig.cpp @@ -17,11 +17,8 @@ #include #include "app_config/AppConfig.h" -#include "common/FileSystemUtil.h" #include "common/Flags.h" -#include "common/JsonUtil.h" #include "common/ParamExtractor.h" -#include "common/YamlUtil.h" #include "pipeline/plugin/PluginRegistry.h" DEFINE_FLAG_BOOL(enable_env_ref_in_config, "enable environment variable reference replacement in configuration", false); @@ -667,61 +664,4 @@ bool PipelineConfig::ReplaceEnvVar() { return res; } -bool LoadConfigDetailFromFile(const filesystem::path& filepath, Json::Value& detail) { - const string& ext = filepath.extension().string(); - const string& configName = filepath.stem().string(); - if (configName == "region_config") { - return false; - } - if (ext != ".yaml" && ext != ".yml" && ext != ".json") { - LOG_WARNING(sLogger, ("unsupported config file format", "skip current object")("filepath", filepath)); - return false; - } - string content; - if (!ReadFile(filepath.string(), content)) { - LOG_WARNING(sLogger, ("failed to open config file", "skip current object")("filepath", filepath)); - return false; - } - if (content.empty()) { - LOG_WARNING(sLogger, ("empty config file", "skip current object")("filepath", filepath)); - return false; - } - string errorMsg; - if (!ParseConfigDetail(content, ext, detail, errorMsg)) { - LOG_WARNING(sLogger, - ("config file format error", "skip current object")("error msg", errorMsg)("filepath", filepath)); - return false; - } - return true; -} - -bool ParseConfigDetail(const string& content, const string& extension, Json::Value& detail, string& errorMsg) { - if (extension == ".json") { - return ParseJsonTable(content, detail, errorMsg); - } else if (extension == ".yaml" || extension == ".yml") { - YAML::Node yamlRoot; - if (!ParseYamlTable(content, yamlRoot, errorMsg)) { - return false; - } - detail = ConvertYamlToJson(yamlRoot); - return true; - } - return false; -} - -bool IsConfigEnabled(const string& name, const Json::Value& detail) { - const char* key = "enable"; - const Json::Value* itr = detail.find(key, key + strlen(key)); - if (itr != nullptr) { - if (!itr->isBool()) { - LOG_WARNING(sLogger, - ("problem encountered in config parsing", - "param enable is not of type bool")("action", "ignore the config")("config", name)); - return false; - } - return itr->asBool(); - } - return true; -} - } // namespace logtail diff --git a/core/config/PipelineConfig.h b/core/config/PipelineConfig.h index 20ff581a33..7d845126e9 100644 --- a/core/config/PipelineConfig.h +++ b/core/config/PipelineConfig.h @@ -17,7 +17,6 @@ #pragma once #include -#include #include #include @@ -84,11 +83,4 @@ inline bool operator!=(const PipelineConfig& lhs, const PipelineConfig& rhs) { return !(lhs == rhs); } -bool LoadConfigDetailFromFile(const std::filesystem::path& filepath, Json::Value& detail); -bool ParseConfigDetail(const std::string& content, - const std::string& extenstion, - Json::Value& detail, - std::string& errorMsg); -bool IsConfigEnabled(const std::string& name, const Json::Value& detail); - } // namespace logtail diff --git a/core/config/TaskConfig.cpp b/core/config/TaskConfig.cpp new file mode 100644 index 0000000000..6fec9ebd19 --- /dev/null +++ b/core/config/TaskConfig.cpp @@ -0,0 +1,52 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "config/TaskConfig.h" + +#include "common/ParamExtractor.h" +#include "task_pipeline/TaskRegistry.h" + +using namespace std; + +namespace logtail { + +bool TaskConfig::Parse() { + string errorMsg; + AlarmManager& alarm = *AlarmManager::GetInstance(); + + if (!GetOptionalUIntParam(*mDetail, "createTime", mCreateTime, errorMsg)) { + TASK_PARAM_WARNING_DEFAULT(sLogger, alarm, errorMsg, mCreateTime, noModule, mName); + } + + auto& detail = (*mDetail)["task"]; + if (!detail.isObject()) { + TASK_PARAM_ERROR_RETURN(sLogger, alarm, "task module is not of type object", noModule, mName); + } + + string key = "Type"; + auto itr = detail.find(key.c_str(), key.c_str() + key.size()); + if (itr == nullptr) { + TASK_PARAM_ERROR_RETURN(sLogger, alarm, "param task.Type is missing", noModule, mName); + } + if (!itr->isString()) { + TASK_PARAM_ERROR_RETURN(sLogger, alarm, "param task.Type is not of type string", noModule, mName); + } + string pluginType = itr->asString(); + if (!TaskRegistry::GetInstance()->IsValidPlugin(pluginType)) { + TASK_PARAM_ERROR_RETURN(sLogger, alarm, "unsupported task plugin", pluginType, mName); + } + return true; +} + +} // namespace logtail diff --git a/core/config/TaskConfig.h b/core/config/TaskConfig.h new file mode 100644 index 0000000000..5fafc176ef --- /dev/null +++ b/core/config/TaskConfig.h @@ -0,0 +1,46 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include +#include + +namespace logtail { + +struct TaskConfig { + std::string mName; + std::unique_ptr mDetail; + uint32_t mCreateTime = 0; + + TaskConfig(const std::string& name, std::unique_ptr&& detail) + : mName(name), mDetail(std::move(detail)) {} + + bool Parse(); +}; + +inline bool operator==(const TaskConfig& lhs, const TaskConfig& rhs) { + return (lhs.mName == rhs.mName) && (*lhs.mDetail == *rhs.mDetail); +} + +inline bool operator!=(const TaskConfig& lhs, const TaskConfig& rhs) { + return !(lhs == rhs); +} + +} // namespace logtail diff --git a/core/config/common_provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp index 58ae7a5340..a843521915 100644 --- a/core/config/common_provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -27,6 +27,7 @@ #include "common/UUIDUtil.h" #include "common/YamlUtil.h" #include "common/version.h" +#include "config/ConfigUtil.h" #include "config/PipelineConfig.h" #include "config/feedbacker/ConfigFeedbackReceiver.h" #include "constants/Constants.h" diff --git a/core/config/provider/ConfigProvider.cpp b/core/config/provider/ConfigProvider.cpp index 402c009ae2..1c6b68b31b 100644 --- a/core/config/provider/ConfigProvider.cpp +++ b/core/config/provider/ConfigProvider.cpp @@ -14,9 +14,9 @@ #include "config/provider/ConfigProvider.h" -#include "InstanceConfigWatcher.h" #include "app_config/AppConfig.h" -#include "config/watcher/ConfigWatcher.h" +#include "config/watcher/InstanceConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" using namespace std; @@ -34,7 +34,7 @@ void ConfigProvider::Init(const string& dir) { error_code ec; filesystem::create_directories(mPipelineSourceDir, ec); - ConfigWatcher::GetInstance()->AddSource(mPipelineSourceDir, &mPipelineMux); + PipelineConfigWatcher::GetInstance()->AddSource(mPipelineSourceDir, &mPipelineMux); ec.clear(); filesystem::create_directories(mInstanceSourceDir, ec); diff --git a/core/config/watcher/ConfigWatcher.cpp b/core/config/watcher/ConfigWatcher.cpp index 2c430a60ed..fae78869b0 100644 --- a/core/config/watcher/ConfigWatcher.cpp +++ b/core/config/watcher/ConfigWatcher.cpp @@ -14,188 +14,10 @@ #include "config/watcher/ConfigWatcher.h" -#include -#include - -#include "PipelineConfig.h" -#include "logger/Logger.h" -#include "pipeline/PipelineManager.h" - using namespace std; namespace logtail { -bool ReadFile(const string& filepath, string& content); - -ConfigWatcher::ConfigWatcher() : mPipelineManager(PipelineManager::GetInstance()) { -} - -PipelineConfigDiff ConfigWatcher::CheckConfigDiff() { - PipelineConfigDiff diff; - unordered_set configSet; - for (const auto& dir : mSourceDir) { - error_code ec; - filesystem::file_status s = filesystem::status(dir, ec); - if (ec) { - LOG_WARNING(sLogger, - ("failed to get config dir path info", "skip current object")("dir path", dir.string())( - "error code", ec.value())("error msg", ec.message())); - continue; - } - if (!filesystem::exists(s)) { - LOG_WARNING(sLogger, ("config dir path not existed", "skip current object")("dir path", dir.string())); - continue; - } - if (!filesystem::is_directory(s)) { - LOG_WARNING(sLogger, - ("config dir path is not a directory", "skip current object")("dir path", dir.string())); - continue; - } - for (auto const& entry : filesystem::directory_iterator(dir, ec)) { - // lock the dir if it is provided by config provider - unique_lock lock; - auto itr = mDirMutexMap.find(dir.string()); - if (itr != mDirMutexMap.end()) { - lock = unique_lock(*itr->second, defer_lock); - lock.lock(); - } - - const filesystem::path& path = entry.path(); - const string& configName = path.stem().string(); - const string& filepath = path.string(); - if (!filesystem::is_regular_file(entry.status(ec))) { - LOG_DEBUG(sLogger, ("config file is not a regular file", "skip current object")("filepath", filepath)); - continue; - } - if (configSet.find(configName) != configSet.end()) { - LOG_WARNING( - sLogger, - ("more than 1 config with the same name is found", "skip current config")("filepath", filepath)); - continue; - } - configSet.insert(configName); - - auto iter = mFileInfoMap.find(filepath); - uintmax_t size = filesystem::file_size(path, ec); - filesystem::file_time_type mTime = filesystem::last_write_time(path, ec); - if (iter == mFileInfoMap.end()) { - mFileInfoMap[filepath] = make_pair(size, mTime); - unique_ptr detail = make_unique(new Json::Value()); - if (!LoadConfigDetailFromFile(path, *detail)) { - continue; - } - if (!IsConfigEnabled(configName, *detail)) { - LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", configName)); - continue; - } - PipelineConfig config(configName, std::move(detail)); - if (!config.Parse()) { - LOG_ERROR(sLogger, ("new config found but invalid", "skip current object")("config", configName)); - AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM, - "new config found but invalid: skip current object, config: " - + configName, - config.mProject, - config.mLogstore, - config.mRegion); - continue; - } - diff.mAdded.push_back(std::move(config)); - LOG_INFO( - sLogger, - ("new config found and passed topology check", "prepare to build pipeline")("config", configName)); - } else if (iter->second.first != size || iter->second.second != mTime) { - // for config currently running, we leave it untouched if new config is invalid - mFileInfoMap[filepath] = make_pair(size, mTime); - unique_ptr detail = make_unique(new Json::Value()); - if (!LoadConfigDetailFromFile(path, *detail)) { - continue; - } - if (!IsConfigEnabled(configName, *detail)) { - if (mPipelineManager->FindConfigByName(configName)) { - diff.mRemoved.push_back(configName); - LOG_INFO(sLogger, - ("existing valid config modified and disabled", - "prepare to stop current running pipeline")("config", configName)); - } else { - LOG_INFO(sLogger, - ("existing invalid config modified and disabled", "skip current object")("config", - configName)); - } - continue; - } - shared_ptr p = mPipelineManager->FindConfigByName(configName); - if (!p) { - PipelineConfig config(configName, std::move(detail)); - if (!config.Parse()) { - LOG_ERROR(sLogger, - ("existing invalid config modified and remains invalid", - "skip current object")("config", configName)); - AlarmManager::GetInstance()->SendAlarm( - CATEGORY_CONFIG_ALARM, - "existing invalid config modified and remains invalid: skip current object, config: " - + configName, - config.mProject, - config.mLogstore, - config.mRegion); - continue; - } - diff.mAdded.push_back(std::move(config)); - LOG_INFO(sLogger, - ("existing invalid config modified and passed topology check", - "prepare to build pipeline")("config", configName)); - } else if (*detail != p->GetConfig()) { - PipelineConfig config(configName, std::move(detail)); - if (!config.Parse()) { - LOG_ERROR(sLogger, - ("existing valid config modified and becomes invalid", - "keep current pipeline running")("config", configName)); - AlarmManager::GetInstance()->SendAlarm( - CATEGORY_CONFIG_ALARM, - "existing valid config modified and becomes invalid: skip current object, config: " - + configName, - config.mProject, - config.mLogstore, - config.mRegion); - continue; - } - diff.mModified.push_back(std::move(config)); - LOG_INFO(sLogger, - ("existing valid config modified and passed topology check", - "prepare to rebuild pipeline")("config", configName)); - } else { - LOG_DEBUG(sLogger, - ("existing valid config file modified, but no change found", "skip current object")); - } - } else { - LOG_DEBUG(sLogger, ("existing config file unchanged", "skip current object")); - } - } - } - for (const auto& name : mPipelineManager->GetAllConfigNames()) { - if (configSet.find(name) == configSet.end()) { - diff.mRemoved.push_back(name); - LOG_INFO(sLogger, - ("existing valid config is removed", "prepare to stop current running pipeline")("config", name)); - } - } - for (const auto& item : mFileInfoMap) { - string configName = filesystem::path(item.first).stem().string(); - if (configSet.find(configName) == configSet.end()) { - mFileInfoMap.erase(item.first); - } - } - - if (!diff.IsEmpty()) { - LOG_INFO(sLogger, - ("config files scan done", "got updates, begin to update pipelines")("added", diff.mAdded.size())( - "modified", diff.mModified.size())("removed", diff.mRemoved.size())); - } else { - LOG_DEBUG(sLogger, ("config files scan done", "no update")); - } - - return diff; -} - void ConfigWatcher::AddSource(const string& dir, mutex* mux) { mSourceDir.emplace_back(dir); if (mux != nullptr) { @@ -203,9 +25,11 @@ void ConfigWatcher::AddSource(const string& dir, mutex* mux) { } } +#ifdef APSARA_UNIT_TEST_MAIN void ConfigWatcher::ClearEnvironment() { mSourceDir.clear(); mFileInfoMap.clear(); } +#endif } // namespace logtail diff --git a/core/config/watcher/ConfigWatcher.h b/core/config/watcher/ConfigWatcher.h index ba58b28f76..0b43d18cbb 100644 --- a/core/config/watcher/ConfigWatcher.h +++ b/core/config/watcher/ConfigWatcher.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 iLogtail Authors + * Copyright 2024 iLogtail Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,43 +16,33 @@ #pragma once -#include #include #include +#include #include #include #include -#include "config/ConfigDiff.h" - namespace logtail { -class PipelineManager; - class ConfigWatcher { public: ConfigWatcher(const ConfigWatcher&) = delete; ConfigWatcher& operator=(const ConfigWatcher&) = delete; - static ConfigWatcher* GetInstance() { - static ConfigWatcher instance; - return &instance; - } - - PipelineConfigDiff CheckConfigDiff(); void AddSource(const std::string& dir, std::mutex* mux = nullptr); - // for ut - void SetPipelineManager(const PipelineManager* pm) { mPipelineManager = pm; } + +#ifdef APSARA_UNIT_TEST_MAIN void ClearEnvironment(); +#endif -private: - ConfigWatcher(); - ~ConfigWatcher() = default; +protected: + ConfigWatcher() = default; + virtual ~ConfigWatcher() = default; std::vector mSourceDir; - std::unordered_map mDirMutexMap; + std::map mDirMutexMap; std::map> mFileInfoMap; - const PipelineManager* mPipelineManager = nullptr; }; -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/config/watcher/InstanceConfigWatcher.cpp b/core/config/watcher/InstanceConfigWatcher.cpp index 6683ac6fab..fcd6e518a3 100644 --- a/core/config/watcher/InstanceConfigWatcher.cpp +++ b/core/config/watcher/InstanceConfigWatcher.cpp @@ -17,16 +17,17 @@ #include #include -#include "config/InstanceConfig.h" +#include "common/FileSystemUtil.h" +#include "config/ConfigUtil.h" +#include "config/InstanceConfigManager.h" #include "logger/Logger.h" using namespace std; namespace logtail { -bool ReadFile(const string& filepath, string& content); - -InstanceConfigWatcher::InstanceConfigWatcher() : mInstanceConfigManager(InstanceConfigManager::GetInstance()) { +InstanceConfigWatcher::InstanceConfigWatcher() + : ConfigWatcher(), mInstanceConfigManager(InstanceConfigManager::GetInstance()) { } InstanceConfigDiff InstanceConfigWatcher::CheckConfigDiff() { @@ -167,16 +168,4 @@ InstanceConfigDiff InstanceConfigWatcher::CheckConfigDiff() { return diff; } -void InstanceConfigWatcher::AddSource(const string& dir, mutex* mux) { - mSourceDir.emplace_back(dir); - if (mux != nullptr) { - mDirMutexMap[dir] = mux; - } -} - -void InstanceConfigWatcher::ClearEnvironment() { - mSourceDir.clear(); - mFileInfoMap.clear(); -} - } // namespace logtail diff --git a/core/config/watcher/InstanceConfigWatcher.h b/core/config/watcher/InstanceConfigWatcher.h index 81a37fce18..c38abec9fb 100644 --- a/core/config/watcher/InstanceConfigWatcher.h +++ b/core/config/watcher/InstanceConfigWatcher.h @@ -1,5 +1,5 @@ /* - * Copyright 2023 iLogtail Authors + * Copyright 2024 iLogtail Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,22 +16,14 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include - -#include "InstanceConfigManager.h" #include "config/ConfigDiff.h" +#include "config/watcher/ConfigWatcher.h" namespace logtail { class InstanceConfigManager; -class InstanceConfigWatcher { +class InstanceConfigWatcher : public ConfigWatcher { public: InstanceConfigWatcher(const InstanceConfigWatcher&) = delete; InstanceConfigWatcher& operator=(const InstanceConfigWatcher&) = delete; @@ -42,19 +34,16 @@ class InstanceConfigWatcher { } InstanceConfigDiff CheckConfigDiff(); - void AddSource(const std::string& dir, std::mutex* mux = nullptr); - // for ut + +#ifdef APSARA_UNIT_TEST_MAIN void SetInstanceConfigManager(const InstanceConfigManager* m) { mInstanceConfigManager = m; } - void ClearEnvironment(); +#endif private: InstanceConfigWatcher(); ~InstanceConfigWatcher() = default; - std::vector mSourceDir; - std::unordered_map mDirMutexMap; - std::map> mFileInfoMap; const InstanceConfigManager* mInstanceConfigManager = nullptr; }; -} // namespace logtail \ No newline at end of file +} // namespace logtail diff --git a/core/config/watcher/PipelineConfigWatcher.cpp b/core/config/watcher/PipelineConfigWatcher.cpp new file mode 100644 index 0000000000..40118cbc47 --- /dev/null +++ b/core/config/watcher/PipelineConfigWatcher.cpp @@ -0,0 +1,315 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "config/watcher/PipelineConfigWatcher.h" + +#include +#include + +#include "common/FileSystemUtil.h" +#include "config/ConfigUtil.h" +#include "logger/Logger.h" +#include "pipeline/PipelineManager.h" +#include "task_pipeline/TaskPipelineManager.h" + +using namespace std; + +namespace logtail { + +PipelineConfigWatcher::PipelineConfigWatcher() + : ConfigWatcher(), + mPipelineManager(PipelineManager::GetInstance()), + mTaskPipelineManager(TaskPipelineManager::GetInstance()) { +} + +pair PipelineConfigWatcher::CheckConfigDiff() { + PipelineConfigDiff pDiff; + TaskConfigDiff tDiff; + unordered_set configSet; + for (const auto& dir : mSourceDir) { + error_code ec; + filesystem::file_status s = filesystem::status(dir, ec); + if (ec) { + LOG_WARNING(sLogger, + ("failed to get config dir path info", "skip current object")("dir path", dir.string())( + "error code", ec.value())("error msg", ec.message())); + continue; + } + if (!filesystem::exists(s)) { + LOG_WARNING(sLogger, ("config dir path not existed", "skip current object")("dir path", dir.string())); + continue; + } + if (!filesystem::is_directory(s)) { + LOG_WARNING(sLogger, + ("config dir path is not a directory", "skip current object")("dir path", dir.string())); + continue; + } + for (auto const& entry : filesystem::directory_iterator(dir, ec)) { + // lock the dir if it is provided by config provider + unique_lock lock; + auto itr = mDirMutexMap.find(dir.string()); + if (itr != mDirMutexMap.end()) { + lock = unique_lock(*itr->second, defer_lock); + lock.lock(); + } + + const filesystem::path& path = entry.path(); + const string& configName = path.stem().string(); + const string& filepath = path.string(); + if (!filesystem::is_regular_file(entry.status(ec))) { + LOG_DEBUG(sLogger, ("config file is not a regular file", "skip current object")("filepath", filepath)); + continue; + } + if (configSet.find(configName) != configSet.end()) { + LOG_WARNING( + sLogger, + ("more than 1 config with the same name is found", "skip current config")("filepath", filepath)); + continue; + } + configSet.insert(configName); + + auto iter = mFileInfoMap.find(filepath); + uintmax_t size = filesystem::file_size(path, ec); + filesystem::file_time_type mTime = filesystem::last_write_time(path, ec); + if (iter == mFileInfoMap.end()) { + mFileInfoMap[filepath] = make_pair(size, mTime); + unique_ptr detail = make_unique(); + if (!LoadConfigDetailFromFile(path, *detail)) { + continue; + } + if (!IsConfigEnabled(configName, *detail)) { + LOG_INFO(sLogger, ("new config found and disabled", "skip current object")("config", configName)); + continue; + } + if (!CheckAddedConfig(configName, std::move(detail), pDiff, tDiff)) { + continue; + } + } else if (iter->second.first != size || iter->second.second != mTime) { + // for config currently running, we leave it untouched if new config is invalid + mFileInfoMap[filepath] = make_pair(size, mTime); + unique_ptr detail = make_unique(); + if (!LoadConfigDetailFromFile(path, *detail)) { + continue; + } + if (!IsConfigEnabled(configName, *detail)) { + switch (GetConfigType(*detail)) { + case ConfigType::Pipeline: + if (mPipelineManager->FindConfigByName(configName)) { + pDiff.mRemoved.push_back(configName); + LOG_INFO(sLogger, + ("existing valid config modified and disabled", + "prepare to stop current running pipeline")("config", configName)); + } else { + LOG_INFO(sLogger, + ("existing invalid config modified and disabled", + "skip current object")("config", configName)); + } + break; + case ConfigType::Task: + if (mTaskPipelineManager->FindPipelineByName(configName)) { + tDiff.mRemoved.push_back(configName); + LOG_INFO(sLogger, + ("existing valid config modified and disabled", + "prepare to stop current running task")("config", configName)); + } else { + LOG_INFO(sLogger, + ("existing invalid config modified and disabled", + "skip current object")("config", configName)); + } + break; + } + continue; + } + if (!CheckModifiedConfig(configName, std::move(detail), pDiff, tDiff)) { + continue; + } + } else { + LOG_DEBUG(sLogger, ("existing config file unchanged", "skip current object")); + } + } + } + for (const auto& name : mPipelineManager->GetAllConfigNames()) { + if (configSet.find(name) == configSet.end()) { + pDiff.mRemoved.push_back(name); + LOG_INFO(sLogger, + ("existing valid config is removed", "prepare to stop current running pipeline")("config", name)); + } + } + for (const auto& name : mTaskPipelineManager->GetAllPipelineNames()) { + if (configSet.find(name) == configSet.end()) { + tDiff.mRemoved.push_back(name); + LOG_INFO(sLogger, + ("existing valid config is removed", "prepare to stop current running task")("config", name)); + } + } + for (auto it = mFileInfoMap.begin(); it != mFileInfoMap.end();) { + string configName = filesystem::path(it->first).stem().string(); + if (configSet.find(configName) == configSet.end()) { + it = mFileInfoMap.erase(it); + } else { + ++it; + } + } + + if (!pDiff.IsEmpty()) { + LOG_INFO(sLogger, + ("config files scan done", "got updates, begin to update pipelines")("added", pDiff.mAdded.size())( + "modified", pDiff.mModified.size())("removed", pDiff.mRemoved.size())); + } else { + LOG_DEBUG(sLogger, ("config files scan done", "no pipeline update")); + } + if (!tDiff.IsEmpty()) { + LOG_INFO(sLogger, + ("config files scan done", "got updates, begin to update tasks")("added", tDiff.mAdded.size())( + "modified", tDiff.mModified.size())("removed", tDiff.mRemoved.size())); + } else { + LOG_DEBUG(sLogger, ("config files scan done", "no task update")); + } + + return make_pair(std::move(pDiff), std::move(tDiff)); +} + +bool PipelineConfigWatcher::CheckAddedConfig(const string& configName, + unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff) { + switch (GetConfigType(*configDetail)) { + case ConfigType::Pipeline: { + PipelineConfig config(configName, std::move(configDetail)); + if (!config.Parse()) { + LOG_ERROR(sLogger, ("new config found but invalid", "skip current object")("config", configName)); + AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM, + "new config found but invalid: skip current object, config: " + + configName, + config.mProject, + config.mLogstore, + config.mRegion); + return false; + } + pDiff.mAdded.push_back(std::move(config)); + LOG_INFO(sLogger, + ("new config found and passed topology check", "prepare to build pipeline")("config", configName)); + break; + } + case ConfigType::Task: { + TaskConfig config(configName, std::move(configDetail)); + if (!config.Parse()) { + LOG_ERROR(sLogger, ("new config found but invalid", "skip current object")("config", configName)); + AlarmManager::GetInstance()->SendAlarm( + CATEGORY_CONFIG_ALARM, "new config found but invalid: skip current object, config: " + configName); + return false; + } + tDiff.mAdded.push_back(std::move(config)); + LOG_INFO(sLogger, + ("new config found and passed topology check", "prepare to build task")("config", configName)); + break; + } + } + return true; +} + +bool PipelineConfigWatcher::CheckModifiedConfig(const string& configName, + unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff) { + switch (GetConfigType(*configDetail)) { + case ConfigType::Pipeline: { + shared_ptr p = mPipelineManager->FindConfigByName(configName); + if (!p) { + PipelineConfig config(configName, std::move(configDetail)); + if (!config.Parse()) { + LOG_ERROR(sLogger, + ("existing invalid config modified and remains invalid", + "skip current object")("config", configName)); + AlarmManager::GetInstance()->SendAlarm( + CATEGORY_CONFIG_ALARM, + "existing invalid config modified and remains invalid: skip current object, config: " + + configName, + config.mProject, + config.mLogstore, + config.mRegion); + return false; + } + pDiff.mAdded.push_back(std::move(config)); + LOG_INFO(sLogger, + ("existing invalid config modified and passed topology check", + "prepare to build pipeline")("config", configName)); + } else if (*configDetail != p->GetConfig()) { + PipelineConfig config(configName, std::move(configDetail)); + if (!config.Parse()) { + LOG_ERROR(sLogger, + ("existing valid config modified and becomes invalid", + "keep current pipeline running")("config", configName)); + AlarmManager::GetInstance()->SendAlarm( + CATEGORY_CONFIG_ALARM, + "existing valid config modified and becomes invalid: skip current object, config: " + + configName, + config.mProject, + config.mLogstore, + config.mRegion); + return false; + } + pDiff.mModified.push_back(std::move(config)); + LOG_INFO(sLogger, + ("existing valid config modified and passed topology check", + "prepare to rebuild pipeline")("config", configName)); + } else { + LOG_DEBUG(sLogger, ("existing valid config file modified, but no change found", "skip current object")); + } + break; + } + case ConfigType::Task: { + auto& p = mTaskPipelineManager->FindPipelineByName(configName); + if (!p) { + TaskConfig config(configName, std::move(configDetail)); + if (!config.Parse()) { + LOG_ERROR(sLogger, + ("existing invalid config modified and remains invalid", + "skip current object")("config", configName)); + AlarmManager::GetInstance()->SendAlarm( + CATEGORY_CONFIG_ALARM, + "existing invalid config modified and remains invalid: skip current object, config: " + + configName); + return false; + } + tDiff.mAdded.push_back(std::move(config)); + LOG_INFO(sLogger, + ("existing invalid config modified and passed topology check", + "prepare to build task")("config", configName)); + } else if (*configDetail != p->GetConfig()) { + TaskConfig config(configName, std::move(configDetail)); + if (!config.Parse()) { + LOG_ERROR(sLogger, + ("existing valid config modified and becomes invalid", + "keep current task running")("config", configName)); + AlarmManager::GetInstance()->SendAlarm( + CATEGORY_CONFIG_ALARM, + "existing valid config modified and becomes invalid: skip current object, config: " + + configName); + return false; + } + tDiff.mModified.push_back(std::move(config)); + LOG_INFO(sLogger, + ("existing valid config modified and passed topology check", + "prepare to rebuild task")("config", configName)); + } else { + LOG_DEBUG(sLogger, ("existing valid config file modified, but no change found", "skip current object")); + } + break; + } + } + return true; +} + +} // namespace logtail diff --git a/core/config/watcher/PipelineConfigWatcher.h b/core/config/watcher/PipelineConfigWatcher.h new file mode 100644 index 0000000000..d1f77967fe --- /dev/null +++ b/core/config/watcher/PipelineConfigWatcher.h @@ -0,0 +1,60 @@ +/* + * Copyright 2023 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "config/ConfigDiff.h" +#include "config/watcher/ConfigWatcher.h" + +namespace logtail { + +class PipelineManager; +class TaskPipelineManager; + +class PipelineConfigWatcher : public ConfigWatcher { +public: + PipelineConfigWatcher(const PipelineConfigWatcher&) = delete; + PipelineConfigWatcher& operator=(const PipelineConfigWatcher&) = delete; + + static PipelineConfigWatcher* GetInstance() { + static PipelineConfigWatcher instance; + return &instance; + } + + std::pair CheckConfigDiff(); + +#ifdef APSARA_UNIT_TEST_MAIN + void SetPipelineManager(const PipelineManager* pm) { mPipelineManager = pm; } +#endif + +private: + PipelineConfigWatcher(); + ~PipelineConfigWatcher() = default; + + bool CheckAddedConfig(const std::string& configName, + std::unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff); + bool CheckModifiedConfig(const std::string& configName, + std::unique_ptr&& configDetail, + PipelineConfigDiff& pDiff, + TaskConfigDiff& tDiff); + + const PipelineManager* mPipelineManager = nullptr; + const TaskPipelineManager* mTaskPipelineManager = nullptr; +}; + +} // namespace logtail diff --git a/core/task_pipeline/Task.h b/core/task_pipeline/Task.h new file mode 100644 index 0000000000..c3d8eedb5b --- /dev/null +++ b/core/task_pipeline/Task.h @@ -0,0 +1,42 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +#include "logger/Logger.h" +#include "monitor/AlarmManager.h" + +namespace logtail { + +class Task { +public: + virtual ~Task() {} + + virtual const std::string& Name() const = 0; + virtual bool Init(const Json::Value& config) = 0; + virtual void Start() = 0; + virtual void Stop(bool isRemoving) = 0; + +protected: + Logger::logger mLogger = sLogger; + AlarmManager* mAlarm = AlarmManager::GetInstance(); +}; + +} // namespace logtail diff --git a/core/task_pipeline/TaskPipeline.cpp b/core/task_pipeline/TaskPipeline.cpp new file mode 100644 index 0000000000..515741e89d --- /dev/null +++ b/core/task_pipeline/TaskPipeline.cpp @@ -0,0 +1,35 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "task_pipeline/TaskPipeline.h" + +#include "task_pipeline/TaskRegistry.h" + +using namespace std; + +namespace logtail { + +bool TaskPipeline::Init(TaskConfig&& config) { + mConfigName = config.mName; + mCreateTime = config.mCreateTime; + mConfig = std::move(config.mDetail); + + const auto& detail = (*mConfig)["task"]; + mPlugin = TaskRegistry::GetInstance()->CreateTask(detail["Type"].asString()); + return mPlugin->Init(detail); +} + +} // namespace logtail diff --git a/core/task_pipeline/TaskPipeline.h b/core/task_pipeline/TaskPipeline.h new file mode 100644 index 0000000000..fc5b95a1f3 --- /dev/null +++ b/core/task_pipeline/TaskPipeline.h @@ -0,0 +1,53 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +#include "config/TaskConfig.h" +#include "task_pipeline/Task.h" + +namespace logtail { + +class TaskPipeline { +public: + const std::string& Name() const { return mConfigName; } + bool Init(TaskConfig&& config); + void Start() { mPlugin->Start(); } + void Stop(bool isRemoving) { mPlugin->Stop(isRemoving); } + const Json::Value& GetConfig() const { return *mConfig; } + +#ifdef APSARA_UNIT_TEST_MAIN + Task* GetPlugin() const { return mPlugin.get(); } +#endif + +private: + std::unique_ptr mPlugin; + + std::string mConfigName; + std::unique_ptr mConfig; + uint32_t mCreateTime = 0; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class TaskPipelineUnittest; +#endif +}; + +} // namespace logtail diff --git a/core/task_pipeline/TaskPipelineManager.cpp b/core/task_pipeline/TaskPipelineManager.cpp new file mode 100644 index 0000000000..a846f1dd4d --- /dev/null +++ b/core/task_pipeline/TaskPipelineManager.cpp @@ -0,0 +1,106 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "task_pipeline/TaskPipelineManager.h" + +#include "config/feedbacker/ConfigFeedbackReceiver.h" + +using namespace std; + +namespace logtail { + +static unique_ptr sEmptyTask; + +void TaskPipelineManager::UpdatePipelines(TaskConfigDiff& diff) { + for (const auto& name : diff.mRemoved) { + auto iter = mPipelineNameEntityMap.find(name); + iter->second->Stop(true); + mPipelineNameEntityMap.erase(iter); + ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(name, ConfigFeedbackStatus::DELETED); + } + for (auto& config : diff.mModified) { + auto p = BuildPipeline(std::move(config)); + if (!p) { + LOG_WARNING( + sLogger, + ("failed to build task for existing config", "keep current task running")("config", config.mName)); + AlarmManager::GetInstance()->SendAlarm( + CATEGORY_CONFIG_ALARM, + "failed to build task for existing config: keep current task running, config: " + config.mName); + ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::FAILED); + continue; + } + LOG_INFO(sLogger, + ("task building for existing config succeeded", + "stop the old task and start the new one")("config", config.mName)); + auto iter = mPipelineNameEntityMap.find(config.mName); + iter->second->Stop(false); + mPipelineNameEntityMap[config.mName] = std::move(p); + mPipelineNameEntityMap[config.mName]->Start(); + ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, ConfigFeedbackStatus::APPLIED); + } + for (auto& config : diff.mAdded) { + auto p = BuildPipeline(std::move(config)); + if (!p) { + LOG_WARNING(sLogger, + ("failed to build task for new config", "skip current object")("config", config.mName)); + AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM, + "failed to build task for new config: skip current object, config: " + + config.mName); + ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::FAILED); + continue; + } + LOG_INFO(sLogger, ("task building for new config succeeded", "begin to start task")("config", config.mName)); + mPipelineNameEntityMap[config.mName] = std::move(p); + mPipelineNameEntityMap[config.mName]->Start(); + ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, ConfigFeedbackStatus::APPLIED); + } +} + +void TaskPipelineManager::StopAllPipelines() { + for (auto& item : mPipelineNameEntityMap) { + item.second->Stop(true); + } + mPipelineNameEntityMap.clear(); +} + +const unique_ptr& TaskPipelineManager::FindPipelineByName(const string& configName) const { + auto it = mPipelineNameEntityMap.find(configName); + if (it != mPipelineNameEntityMap.end()) { + return it->second; + } + return sEmptyTask; +} + +vector TaskPipelineManager::GetAllPipelineNames() const { + vector res; + for (const auto& item : mPipelineNameEntityMap) { + res.push_back(item.first); + } + return res; +} + +unique_ptr TaskPipelineManager::BuildPipeline(TaskConfig&& config) { + auto p = make_unique(); + if (!p->Init(std::move(config))) { + return nullptr; + } + return p; +} + +} // namespace logtail diff --git a/core/task_pipeline/TaskPipelineManager.h b/core/task_pipeline/TaskPipelineManager.h new file mode 100644 index 0000000000..60776f6fd4 --- /dev/null +++ b/core/task_pipeline/TaskPipelineManager.h @@ -0,0 +1,62 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "config/ConfigDiff.h" +#include "config/TaskConfig.h" +#include "task_pipeline/TaskPipeline.h" + +namespace logtail { + +class TaskPipelineManager { +public: + TaskPipelineManager(const TaskPipelineManager&) = delete; + TaskPipelineManager& operator=(const TaskPipelineManager&) = delete; + + static TaskPipelineManager* GetInstance() { + static TaskPipelineManager instance; + return &instance; + } + + void UpdatePipelines(TaskConfigDiff& diff); + void StopAllPipelines(); + const std::unique_ptr& FindPipelineByName(const std::string& configName) const; + std::vector GetAllPipelineNames() const; + +#ifdef APSARA_UNIT_TEST_MAIN + void ClearEnvironment() { mPipelineNameEntityMap.clear(); } +#endif + +private: + TaskPipelineManager() = default; + ~TaskPipelineManager() = default; + + std::unique_ptr BuildPipeline(TaskConfig&& config); + + std::unordered_map> mPipelineNameEntityMap; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class TaskPipelineManagerUnittest; +#endif +}; + +} // namespace logtail diff --git a/core/task_pipeline/TaskRegistry.cpp b/core/task_pipeline/TaskRegistry.cpp new file mode 100644 index 0000000000..b55132e4d4 --- /dev/null +++ b/core/task_pipeline/TaskRegistry.cpp @@ -0,0 +1,45 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "task_pipeline/TaskRegistry.h" + +using namespace std; + +namespace logtail { + +void TaskRegistry::LoadPlugins() { + // RegisterCreator(TaskMock::sName, []() { return make_unique(); }); +} + +void TaskRegistry::UnloadPlugins() { + mPluginDict.clear(); +} + +unique_ptr TaskRegistry::CreateTask(const std::string& name) { + auto it = mPluginDict.find(name); + if (it == mPluginDict.end()) { + return nullptr; + } + return it->second(); +} + +bool TaskRegistry::IsValidPlugin(const std::string& name) const { + return mPluginDict.find(name) != mPluginDict.end(); +} + +void TaskRegistry::RegisterCreator(const std::string& name, std::function()>&& creator) { + mPluginDict[name] = std::move(creator); +} + +} // namespace logtail diff --git a/core/task_pipeline/TaskRegistry.h b/core/task_pipeline/TaskRegistry.h new file mode 100644 index 0000000000..26c70a7a71 --- /dev/null +++ b/core/task_pipeline/TaskRegistry.h @@ -0,0 +1,57 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "task_pipeline/Task.h" + +namespace logtail { + +class TaskRegistry { +public: + TaskRegistry(const TaskRegistry&) = delete; + TaskRegistry& operator=(const TaskRegistry&) = delete; + + static TaskRegistry* GetInstance() { + static TaskRegistry instance; + return &instance; + } + + void LoadPlugins(); + void UnloadPlugins(); + std::unique_ptr CreateTask(const std::string& name); + bool IsValidPlugin(const std::string& name) const; + +private: + TaskRegistry() = default; + ~TaskRegistry() = default; + + void RegisterCreator(const std::string& name, std::function()>&& creator); + + std::unordered_map()>> mPluginDict; + +#ifdef APSARA_UNIT_TEST_MAIN + friend class TaskConfigUnittest; + friend void LoadTaskMock(); +#endif +}; + +} // namespace logtail diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 6b41223ab5..17d62a4e53 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -54,6 +54,7 @@ macro(add_core_subdir) add_subdirectory(serializer) add_subdirectory(prometheus) add_subdirectory(route) + add_subdirectory(task_pipeline) endmacro() macro(add_spl_subdir) diff --git a/core/unittest/config/CMakeLists.txt b/core/unittest/config/CMakeLists.txt index 823419036e..560d47d393 100644 --- a/core/unittest/config/CMakeLists.txt +++ b/core/unittest/config/CMakeLists.txt @@ -21,8 +21,11 @@ project(config_unittest) # add_executable(config_updator_unittest ConfigUpdatorUnittest.cpp) # target_link_libraries(config_updator_unittest ${UT_BASE_TARGET}) -add_executable(config_unittest PipelineConfigUnittest.cpp) -target_link_libraries(config_unittest ${UT_BASE_TARGET}) +add_executable(pipeline_config_unittest PipelineConfigUnittest.cpp) +target_link_libraries(pipeline_config_unittest ${UT_BASE_TARGET}) + +add_executable(task_config_unittest TaskConfigUnittest.cpp) +target_link_libraries(task_config_unittest ${UT_BASE_TARGET}) add_executable(config_watcher_unittest ConfigWatcherUnittest.cpp) target_link_libraries(config_watcher_unittest ${UT_BASE_TARGET}) @@ -42,7 +45,8 @@ add_executable(common_config_provider_unittest CommonConfigProviderUnittest.cpp) target_link_libraries(common_config_provider_unittest ${UT_BASE_TARGET}) include(GoogleTest) -gtest_discover_tests(config_unittest) +gtest_discover_tests(pipeline_config_unittest) +gtest_discover_tests(task_config_unittest) gtest_discover_tests(config_watcher_unittest) gtest_discover_tests(config_update_unittest) if (ENABLE_ENTERPRISE) diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index 276c6a14ef..1fd43d0652 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -21,7 +21,7 @@ #include "config/ConfigDiff.h" #include "config/InstanceConfigManager.h" #include "config/common_provider/CommonConfigProvider.h" -#include "config/watcher/ConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" #include "config/watcher/InstanceConfigWatcher.h" #include "gmock/gmock.h" #include "monitor/Monitor.h" @@ -432,18 +432,18 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { APSARA_TEST_EQUAL(provider.mPipelineConfigInfoMap["config2"].status, ConfigFeedbackStatus::FAILED); // 处理 pipelineconfig - PipelineConfigDiff pipelineConfigDiff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff); - APSARA_TEST_TRUE(!pipelineConfigDiff.IsEmpty()); - APSARA_TEST_EQUAL(1U, pipelineConfigDiff.mAdded.size()); - APSARA_TEST_EQUAL(pipelineConfigDiff.mAdded[0].mName, "config1"); + auto pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); + APSARA_TEST_TRUE(!pipelineConfigDiff.first.IsEmpty()); + APSARA_TEST_EQUAL(1U, pipelineConfigDiff.first.mAdded.size()); + APSARA_TEST_EQUAL(pipelineConfigDiff.first.mAdded[0].mName, "config1"); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames().size(), 1); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], "config1"); // 再次处理 pipelineconfig - pipelineConfigDiff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff); - APSARA_TEST_TRUE(pipelineConfigDiff.IsEmpty()); - APSARA_TEST_TRUE(pipelineConfigDiff.mAdded.empty()); + pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); + APSARA_TEST_TRUE(pipelineConfigDiff.first.IsEmpty()); + APSARA_TEST_TRUE(pipelineConfigDiff.first.mAdded.empty()); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames().size(), 1); APSARA_TEST_EQUAL(PipelineManager::GetInstance()->GetAllConfigNames()[0], "config1"); @@ -648,17 +648,17 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { APSARA_TEST_TRUE(provider.mPipelineConfigInfoMap.empty()); // 处理pipelineConfigDiff - PipelineConfigDiff pipelineConfigDiff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff); - APSARA_TEST_TRUE(!pipelineConfigDiff.IsEmpty()); - APSARA_TEST_EQUAL(1U, pipelineConfigDiff.mRemoved.size()); - APSARA_TEST_EQUAL(pipelineConfigDiff.mRemoved[0], "config1"); + auto pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); + APSARA_TEST_TRUE(!pipelineConfigDiff.first.IsEmpty()); + APSARA_TEST_EQUAL(1U, pipelineConfigDiff.first.mRemoved.size()); + APSARA_TEST_EQUAL(pipelineConfigDiff.first.mRemoved[0], "config1"); APSARA_TEST_TRUE(PipelineManager::GetInstance()->GetAllConfigNames().empty()); // 再次处理pipelineConfigDiff - pipelineConfigDiff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff); - APSARA_TEST_TRUE(pipelineConfigDiff.IsEmpty()); - APSARA_TEST_TRUE(pipelineConfigDiff.mRemoved.empty()); + pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManager::GetInstance()->UpdatePipelines(pipelineConfigDiff.first); + APSARA_TEST_TRUE(pipelineConfigDiff.first.IsEmpty()); + APSARA_TEST_TRUE(pipelineConfigDiff.first.mRemoved.empty()); APSARA_TEST_TRUE(PipelineManager::GetInstance()->GetAllConfigNames().empty()); APSARA_TEST_TRUE(provider.mInstanceConfigInfoMap.empty()); diff --git a/core/unittest/config/ConfigUpdateUnittest.cpp b/core/unittest/config/ConfigUpdateUnittest.cpp index 45b6fc4c7f..c72f645e1a 100644 --- a/core/unittest/config/ConfigUpdateUnittest.cpp +++ b/core/unittest/config/ConfigUpdateUnittest.cpp @@ -19,11 +19,13 @@ #include #include "config/PipelineConfig.h" -#include "config/watcher/ConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" #include "pipeline/Pipeline.h" #include "pipeline/PipelineManager.h" #include "pipeline/plugin/PluginRegistry.h" +#include "task_pipeline/TaskPipelineManager.h" #include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" using namespace std; @@ -74,19 +76,24 @@ class ConfigUpdateUnittest : public testing::Test { protected: static void SetUpTestCase() { PluginRegistry::GetInstance()->LoadPlugins(); - ConfigWatcher::GetInstance()->SetPipelineManager(PipelineManagerMock::GetInstance()); + LoadTaskMock(); + PipelineConfigWatcher::GetInstance()->SetPipelineManager(PipelineManagerMock::GetInstance()); } - static void TearDownTestCase() { PluginRegistry::GetInstance()->UnloadPlugins(); } + static void TearDownTestCase() { + PluginRegistry::GetInstance()->UnloadPlugins(); + TaskRegistry::GetInstance()->UnloadPlugins(); + } void SetUp() override { filesystem::create_directories(configDir); - ConfigWatcher::GetInstance()->AddSource(configDir.string()); + PipelineConfigWatcher::GetInstance()->AddSource(configDir.string()); } void TearDown() override { PipelineManagerMock::GetInstance()->ClearEnvironment(); - ConfigWatcher::GetInstance()->ClearEnvironment(); + TaskPipelineManager::GetInstance()->ClearEnvironment(); + PipelineConfigWatcher::GetInstance()->ClearEnvironment(); filesystem::remove_all(configDir); } @@ -94,13 +101,17 @@ class ConfigUpdateUnittest : public testing::Test { void PrepareInitialSettings() const; void GenerateInitialConfigs() const; - filesystem::path configDir = "./config"; - vector configPaths = {configDir / "invalid_format.json", - configDir / "invalid_detail.json", - configDir / "enabled_valid.json", - configDir / "disabled_valid.json"}; - const string invalidConfigWithInvalidFormat = R"({"inputs":{}})"; - const string invalidConfigWithInvalidDetail = R"( + filesystem::path configDir = "./pipeline_config"; + vector pipelineConfigPaths = {configDir / "pipeline_invalid_format.json", + configDir / "pipeline_invalid_detail.json", + configDir / "pipeline_enabled_valid.json", + configDir / "pipeline_disabled_valid.json"}; + vector taskConfigPaths = {configDir / "task_invalid_format.json", + configDir / "task_invalid_detail.json", + configDir / "task_enabled_valid.json", + configDir / "task_disabled_valid.json"}; + const string invalidPipelineConfigWithInvalidFormat = R"({"inputs":{}})"; + const string invalidPipelineConfigWithInvalidDetail = R"( { "valid": false, "inputs": [ @@ -115,7 +126,7 @@ class ConfigUpdateUnittest : public testing::Test { ] } )"; - const string enabledValidConfig = R"( + const string enabledValidPipelineConfig = R"( { "valid": true, "inputs": [ @@ -130,7 +141,7 @@ class ConfigUpdateUnittest : public testing::Test { ] } )"; - const string disabledValidConfig = R"( + const string disabledValidPipelineConfig = R"( { "valid": true, "enable": false, @@ -147,8 +158,8 @@ class ConfigUpdateUnittest : public testing::Test { } )"; - const string newInvalidConfigWithInvalidFormat = R"({"flushers":{}})"; - const string newInvalidConfigWithInvalidDetail = R"( + const string newInvalidPipelineConfigWithInvalidFormat = R"({"flushers":{}})"; + const string newInvalidPipelineConfigWithInvalidDetail = R"( { "valid": false, "inputs": [ @@ -163,7 +174,7 @@ class ConfigUpdateUnittest : public testing::Test { ] } )"; - const string newEnabledValidConfig = R"( + const string newEnabledValidPipelineConfig = R"( { "valid": true, "inputs": [ @@ -178,7 +189,7 @@ class ConfigUpdateUnittest : public testing::Test { ] } )"; - const string newDisabledValidConfig = R"( + const string newDisabledValidPipelineConfig = R"( { "valid": true, "enable": false, @@ -192,182 +203,386 @@ class ConfigUpdateUnittest : public testing::Test { "Type": "flusher_sls" } ] +} + )"; + + const string invalidTaskConfigWithInvalidFormat = R"({"task":[]})"; + const string invalidTaskConfigWithInvalidDetail = R"( +{ + "task": { + "Type": "task_mock", + "Valid": false + } +} + )"; + const string enabledValidTaskConfig = R"( +{ + "task": { + "Type": "task_mock" + } +} + )"; + const string disabledValidTaskConfig = R"( +{ + "enable": false, + "task": { + "Type": "task_mock" + } +} + )"; + + const string newInvalidTaskConfigWithInvalidFormat = R"({"task":""})"; + const string newInvalidTaskConfigWithInvalidDetail = R"( +{ + "task": { + "Type": "task_mock", + "Valid": false, + "Other": true + } +} + )"; + const string newEnabledValidTaskConfig = R"( +{ + "task": { + "Type": "task_mock", + "Other": true + } +} + )"; + const string newDisabledValidTaskConfig = R"( +{ + "enable": false, + "task": { + "Type": "task_mock", + "Other": true + } } )"; }; void ConfigUpdateUnittest::OnStartUp() const { - PipelineConfigDiff diff; - diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_TRUE(diff.IsEmpty()); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_TRUE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.second.IsEmpty()); GenerateInitialConfigs(); - diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_EQUAL(2U, diff.mAdded.size()); - APSARA_TEST_TRUE(diff.mModified.empty()); - APSARA_TEST_TRUE(diff.mRemoved.empty()); - - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_EQUAL(2U, diff.first.mAdded.size()); + APSARA_TEST_TRUE(diff.first.mModified.empty()); + APSARA_TEST_TRUE(diff.first.mRemoved.empty()); + APSARA_TEST_FALSE(diff.second.IsEmpty()); + APSARA_TEST_EQUAL(2U, diff.second.mAdded.size()); + APSARA_TEST_TRUE(diff.second.mModified.empty()); + APSARA_TEST_TRUE(diff.second.mRemoved.empty()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + APSARA_TEST_NOT_EQUAL(nullptr, ptr); + APSARA_TEST_EQUAL(TaskMock::sName, ptr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(ptr->GetPlugin())->mIsRunning); } void ConfigUpdateUnittest::OnConfigDelete() const { PrepareInitialSettings(); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); filesystem::remove_all(configDir); - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_TRUE(diff.mAdded.empty()); - APSARA_TEST_TRUE(diff.mModified.empty()); - APSARA_TEST_EQUAL(1U, diff.mRemoved.size()); - - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.first.mAdded.empty()); + APSARA_TEST_TRUE(diff.first.mModified.empty()); + APSARA_TEST_EQUAL(1U, diff.first.mRemoved.size()); + APSARA_TEST_FALSE(diff.second.IsEmpty()); + APSARA_TEST_TRUE(diff.second.mAdded.empty()); + APSARA_TEST_TRUE(diff.second.mModified.empty()); + APSARA_TEST_EQUAL(1U, diff.second.mRemoved.size()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_TRUE(PipelineManagerMock::GetInstance()->GetAllConfigNames().empty()); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); + APSARA_TEST_TRUE(TaskPipelineManager::GetInstance()->GetAllPipelineNames().empty()); } void ConfigUpdateUnittest::OnConfigToInvalidFormat() const { PrepareInitialSettings(); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); - for (const auto& path : configPaths) { + for (const auto& path : pipelineConfigPaths) { + ofstream fout(path, ios::trunc); + fout << newInvalidPipelineConfigWithInvalidFormat; + } + for (const auto& path : taskConfigPaths) { ofstream fout(path, ios::trunc); - fout << newInvalidConfigWithInvalidFormat; + fout << newInvalidTaskConfigWithInvalidFormat; } - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_TRUE(diff.IsEmpty()); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_TRUE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.second.IsEmpty()); } void ConfigUpdateUnittest::OnConfigToInvalidDetail() const { PrepareInitialSettings(); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); - for (const auto& path : configPaths) { + for (const auto& path : pipelineConfigPaths) { ofstream fout(path, ios::trunc); - fout << newInvalidConfigWithInvalidDetail; + fout << newInvalidPipelineConfigWithInvalidDetail; } - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_EQUAL(3U, diff.mAdded.size()); - APSARA_TEST_EQUAL(1U, diff.mModified.size()); - APSARA_TEST_TRUE(diff.mRemoved.empty()); - - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + for (const auto& path : taskConfigPaths) { + ofstream fout(path, ios::trunc); + fout << newInvalidTaskConfigWithInvalidDetail; + } + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_EQUAL(3U, diff.first.mAdded.size()); + APSARA_TEST_EQUAL(1U, diff.first.mModified.size()); + APSARA_TEST_TRUE(diff.first.mRemoved.empty()); + APSARA_TEST_FALSE(diff.second.IsEmpty()); + APSARA_TEST_EQUAL(3U, diff.second.mAdded.size()); + APSARA_TEST_EQUAL(1U, diff.second.mModified.size()); + APSARA_TEST_TRUE(diff.second.mRemoved.empty()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); + auto& newPtr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + APSARA_TEST_EQUAL(ptr, newPtr); + APSARA_TEST_EQUAL(TaskMock::sName, newPtr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(newPtr->GetPlugin())->mIsRunning); } void ConfigUpdateUnittest::OnConfigToEnabledValid() const { PrepareInitialSettings(); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); - for (const auto& path : configPaths) { + for (const auto& path : pipelineConfigPaths) { ofstream fout(path, ios::trunc); - fout << newEnabledValidConfig; + fout << newEnabledValidPipelineConfig; } - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_EQUAL(3U, diff.mAdded.size()); - APSARA_TEST_EQUAL(1U, diff.mModified.size()); - APSARA_TEST_TRUE(diff.mRemoved.empty()); - - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + for (const auto& path : taskConfigPaths) { + ofstream fout(path, ios::trunc); + fout << newEnabledValidTaskConfig; + } + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_EQUAL(3U, diff.first.mAdded.size()); + APSARA_TEST_EQUAL(1U, diff.first.mModified.size()); + APSARA_TEST_TRUE(diff.first.mRemoved.empty()); + APSARA_TEST_FALSE(diff.second.IsEmpty()); + APSARA_TEST_EQUAL(3U, diff.second.mAdded.size()); + APSARA_TEST_EQUAL(1U, diff.second.mModified.size()); + APSARA_TEST_TRUE(diff.second.mRemoved.empty()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_EQUAL(4U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); + APSARA_TEST_EQUAL(4U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); + { + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_invalid_format"); + APSARA_TEST_NOT_EQUAL(nullptr, ptr); + APSARA_TEST_EQUAL(TaskMock::sName, ptr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(ptr->GetPlugin())->mIsRunning); + } + { + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_invalid_detail"); + APSARA_TEST_NOT_EQUAL(nullptr, ptr); + APSARA_TEST_EQUAL(TaskMock::sName, ptr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(ptr->GetPlugin())->mIsRunning); + } + { + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + APSARA_TEST_NOT_EQUAL(nullptr, ptr); + APSARA_TEST_EQUAL(TaskMock::sName, ptr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(ptr->GetPlugin())->mIsRunning); + } + { + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_disabled_valid"); + APSARA_TEST_NOT_EQUAL(nullptr, ptr); + APSARA_TEST_EQUAL(TaskMock::sName, ptr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(ptr->GetPlugin())->mIsRunning); + } } void ConfigUpdateUnittest::OnConfigToDisabledValid() const { PrepareInitialSettings(); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); - for (const auto& path : configPaths) { + for (const auto& path : pipelineConfigPaths) { ofstream fout(path, ios::trunc); - fout << newDisabledValidConfig; + fout << newDisabledValidPipelineConfig; } - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_TRUE(diff.mAdded.empty()); - APSARA_TEST_TRUE(diff.mModified.empty()); - APSARA_TEST_EQUAL(1U, diff.mRemoved.size()); + for (const auto& path : taskConfigPaths) { + ofstream fout(path, ios::trunc); + fout << newDisabledValidTaskConfig; + } + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.first.mAdded.empty()); + APSARA_TEST_TRUE(diff.first.mModified.empty()); + APSARA_TEST_EQUAL(1U, diff.first.mRemoved.size()); - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_TRUE(PipelineManagerMock::GetInstance()->GetAllConfigNames().empty()); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); + APSARA_TEST_TRUE(TaskPipelineManager::GetInstance()->GetAllPipelineNames().empty()); } void ConfigUpdateUnittest::OnConfigUnchanged() const { PrepareInitialSettings(); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_TRUE(diff.IsEmpty()); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_TRUE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.second.IsEmpty()); GenerateInitialConfigs(); // mandatorily overwrite modify time in case of no update when file content remains the same. - for (const auto& path : configPaths) { + for (const auto& path : pipelineConfigPaths) { filesystem::file_time_type fTime = filesystem::last_write_time(path); filesystem::last_write_time(path, fTime + 1s); } - diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_EQUAL(1U, diff.mAdded.size()); - APSARA_TEST_TRUE(diff.mModified.empty()); - APSARA_TEST_TRUE(diff.mRemoved.empty()); - - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + for (const auto& path : taskConfigPaths) { + filesystem::file_time_type fTime = filesystem::last_write_time(path); + filesystem::last_write_time(path, fTime + 1s); + } + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_EQUAL(1U, diff.first.mAdded.size()); + APSARA_TEST_TRUE(diff.first.mModified.empty()); + APSARA_TEST_TRUE(diff.first.mRemoved.empty()); + APSARA_TEST_FALSE(diff.second.IsEmpty()); + APSARA_TEST_EQUAL(1U, diff.second.mAdded.size()); + APSARA_TEST_TRUE(diff.second.mModified.empty()); + APSARA_TEST_TRUE(diff.second.mRemoved.empty()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); + auto& newPtr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + APSARA_TEST_EQUAL(ptr, newPtr); + APSARA_TEST_EQUAL(TaskMock::sName, newPtr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(newPtr->GetPlugin())->mIsRunning); } void ConfigUpdateUnittest::OnConfigAdded() const { PrepareInitialSettings(); APSARA_TEST_EQUAL(1U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + APSARA_TEST_EQUAL(1U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); { - ofstream fout(configDir / "add_invalid_format.json", ios::trunc); - fout << invalidConfigWithInvalidFormat; + ofstream fout(configDir / "add_pipeline_invalid_format.json", ios::trunc); + fout << invalidPipelineConfigWithInvalidFormat; } { - ofstream fout(configDir / "add_invalid_detail.json", ios::trunc); - fout << invalidConfigWithInvalidDetail; + ofstream fout(configDir / "add_pipeline_invalid_detail.json", ios::trunc); + fout << invalidPipelineConfigWithInvalidDetail; } { - ofstream fout(configDir / "add_enabled_valid.json", ios::trunc); - fout << enabledValidConfig; + ofstream fout(configDir / "add_pipeline_enabled_valid.json", ios::trunc); + fout << enabledValidPipelineConfig; } { - ofstream fout(configDir / "add_disabled_valid.json", ios::trunc); - fout << disabledValidConfig; + ofstream fout(configDir / "add_pipeline_disabled_valid.json", ios::trunc); + fout << disabledValidPipelineConfig; } - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_EQUAL(2U, diff.mAdded.size()); - APSARA_TEST_TRUE(diff.mModified.empty()); - APSARA_TEST_TRUE(diff.mRemoved.empty()); - - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + { + ofstream fout(configDir / "add_task_invalid_format.json", ios::trunc); + fout << invalidTaskConfigWithInvalidFormat; + } + { + ofstream fout(configDir / "add_task_invalid_detail.json", ios::trunc); + fout << invalidTaskConfigWithInvalidDetail; + } + { + ofstream fout(configDir / "add_task_enabled_valid.json", ios::trunc); + fout << enabledValidTaskConfig; + } + { + ofstream fout(configDir / "add_task_disabled_valid.json", ios::trunc); + fout << disabledValidTaskConfig; + } + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_EQUAL(2U, diff.first.mAdded.size()); + APSARA_TEST_TRUE(diff.first.mModified.empty()); + APSARA_TEST_TRUE(diff.first.mRemoved.empty()); + APSARA_TEST_FALSE(diff.second.IsEmpty()); + APSARA_TEST_EQUAL(2U, diff.second.mAdded.size()); + APSARA_TEST_TRUE(diff.second.mModified.empty()); + APSARA_TEST_TRUE(diff.second.mRemoved.empty()); + + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); APSARA_TEST_EQUAL(2U, PipelineManagerMock::GetInstance()->GetAllConfigNames().size()); + auto& ptr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); + APSARA_TEST_EQUAL(2U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); + { + auto& newPtr = TaskPipelineManager::GetInstance()->FindPipelineByName("task_enabled_valid"); + APSARA_TEST_EQUAL(ptr, newPtr); + APSARA_TEST_EQUAL(TaskMock::sName, newPtr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(newPtr->GetPlugin())->mIsRunning); + } + { + auto& newPtr = TaskPipelineManager::GetInstance()->FindPipelineByName("add_task_enabled_valid"); + APSARA_TEST_NOT_EQUAL(nullptr, newPtr); + APSARA_TEST_EQUAL(TaskMock::sName, newPtr->GetPlugin()->Name()); + APSARA_TEST_TRUE(static_cast(newPtr->GetPlugin())->mIsRunning); + } } void ConfigUpdateUnittest::PrepareInitialSettings() const { GenerateInitialConfigs(); - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - PipelineManagerMock::GetInstance()->UpdatePipelines(diff); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + PipelineManagerMock::GetInstance()->UpdatePipelines(diff.first); + TaskPipelineManager::GetInstance()->UpdatePipelines(diff.second); } void ConfigUpdateUnittest::GenerateInitialConfigs() const { { - ofstream fout(configPaths[0], ios::trunc); - fout << invalidConfigWithInvalidFormat; + ofstream fout(pipelineConfigPaths[0], ios::trunc); + fout << invalidPipelineConfigWithInvalidFormat; + } + { + ofstream fout(pipelineConfigPaths[1], ios::trunc); + fout << invalidPipelineConfigWithInvalidDetail; + } + { + ofstream fout(pipelineConfigPaths[2], ios::trunc); + fout << enabledValidPipelineConfig; + } + { + ofstream fout(pipelineConfigPaths[3], ios::trunc); + fout << disabledValidPipelineConfig; + } + { + ofstream fout(taskConfigPaths[0], ios::trunc); + fout << invalidTaskConfigWithInvalidFormat; } { - ofstream fout(configPaths[1], ios::trunc); - fout << invalidConfigWithInvalidDetail; + ofstream fout(taskConfigPaths[1], ios::trunc); + fout << invalidTaskConfigWithInvalidDetail; } { - ofstream fout(configPaths[2], ios::trunc); - fout << enabledValidConfig; + ofstream fout(taskConfigPaths[2], ios::trunc); + fout << enabledValidTaskConfig; } { - ofstream fout(configPaths[3], ios::trunc); - fout << disabledValidConfig; + ofstream fout(taskConfigPaths[3], ios::trunc); + fout << disabledValidTaskConfig; } } diff --git a/core/unittest/config/ConfigWatcherUnittest.cpp b/core/unittest/config/ConfigWatcherUnittest.cpp index 20df15856f..4edec0c4f2 100644 --- a/core/unittest/config/ConfigWatcherUnittest.cpp +++ b/core/unittest/config/ConfigWatcherUnittest.cpp @@ -16,7 +16,7 @@ #include #include "config/ConfigDiff.h" -#include "config/watcher/ConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" #include "config/watcher/InstanceConfigWatcher.h" #include "pipeline/plugin/PluginRegistry.h" #include "unittest/Unittest.h" @@ -33,11 +33,11 @@ class ConfigWatcherUnittest : public testing::Test { protected: void SetUp() override { - ConfigWatcher::GetInstance()->AddSource(configDir.string()); + PipelineConfigWatcher::GetInstance()->AddSource(configDir.string()); InstanceConfigWatcher::GetInstance()->AddSource(instanceConfigDir.string()); } - void TearDown() override { ConfigWatcher::GetInstance()->ClearEnvironment(); } + void TearDown() override { PipelineConfigWatcher::GetInstance()->ClearEnvironment(); } private: static const filesystem::path configDir; @@ -49,13 +49,15 @@ const filesystem::path ConfigWatcherUnittest::instanceConfigDir = "./instance_co void ConfigWatcherUnittest::InvalidConfigDirFound() const { { - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_TRUE(diff.IsEmpty()); - - { ofstream fout("config"); } - diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_TRUE(diff.IsEmpty()); - filesystem::remove_all("config"); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_TRUE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.second.IsEmpty()); + + { ofstream fout("pipeline_config"); } + diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_TRUE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.second.IsEmpty()); + filesystem::remove_all("pipeline_config"); } { InstanceConfigDiff diff = InstanceConfigWatcher::GetInstance()->CheckConfigDiff(); @@ -79,8 +81,9 @@ void ConfigWatcherUnittest::InvalidConfigFileFound() const { ofstream fout(configDir / "invalid_format.json"); fout << "[}"; } - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_TRUE(diff.IsEmpty()); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_TRUE(diff.first.IsEmpty()); + APSARA_TEST_TRUE(diff.second.IsEmpty()); filesystem::remove_all(configDir); } { @@ -102,8 +105,8 @@ void ConfigWatcherUnittest::InvalidConfigFileFound() const { void ConfigWatcherUnittest::DuplicateConfigs() const { { PluginRegistry::GetInstance()->LoadPlugins(); - ConfigWatcher::GetInstance()->AddSource("dir1"); - ConfigWatcher::GetInstance()->AddSource("dir2"); + PipelineConfigWatcher::GetInstance()->AddSource("dir1"); + PipelineConfigWatcher::GetInstance()->AddSource("dir2"); filesystem::create_directories("config"); filesystem::create_directories("dir1"); @@ -127,9 +130,9 @@ void ConfigWatcherUnittest::DuplicateConfigs() const { )"; } { ofstream fout("dir2/config.json"); } - PipelineConfigDiff diff = ConfigWatcher::GetInstance()->CheckConfigDiff(); - APSARA_TEST_FALSE(diff.IsEmpty()); - APSARA_TEST_EQUAL(1U, diff.mAdded.size()); + auto diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); + APSARA_TEST_FALSE(diff.first.IsEmpty()); + APSARA_TEST_EQUAL(1U, diff.first.mAdded.size()); filesystem::remove_all("dir1"); filesystem::remove_all("dir2"); diff --git a/core/unittest/config/TaskConfigUnittest.cpp b/core/unittest/config/TaskConfigUnittest.cpp new file mode 100644 index 0000000000..658b459914 --- /dev/null +++ b/core/unittest/config/TaskConfigUnittest.cpp @@ -0,0 +1,151 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include + +#include "common/JsonUtil.h" +#include "config/TaskConfig.h" +#include "task_pipeline/TaskRegistry.h" +#include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" + +using namespace std; + +namespace logtail { + +class TaskConfigUnittest : public testing::Test { +public: + void HandleValidConfig() const; + void HandleInvalidCreateTime() const; + void HandleInvalidTask() const; + +protected: + static void SetUpTestCase() { LoadTaskMock(); } + static void TearDownTestCase() { TaskRegistry::GetInstance()->UnloadPlugins(); } + +private: + const string configName = "test"; +}; + +void TaskConfigUnittest::HandleValidConfig() const { + unique_ptr configJson; + string configStr, errorMsg; + unique_ptr config; + + configStr = R"( + { + "createTime": 1234567890, + "task": { + "Type": "task_mock" + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_TRUE(config->Parse()); + APSARA_TEST_EQUAL(configName, config->mName); + APSARA_TEST_EQUAL(1234567890U, config->mCreateTime); +} + +void TaskConfigUnittest::HandleInvalidCreateTime() const { + unique_ptr configJson; + string configStr, errorMsg; + unique_ptr config; + + configStr = R"( + { + "createTime": "1234567890", + "task": { + "Type": "task_mock" + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_TRUE(config->Parse()); + APSARA_TEST_EQUAL(0U, config->mCreateTime); +} + +void TaskConfigUnittest::HandleInvalidTask() const { + unique_ptr configJson; + string configStr, errorMsg; + unique_ptr config; + + // task is not of type object + configStr = R"( + { + "createTime": "123456789", + "task": [] + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_FALSE(config->Parse()); + + // no Type + configStr = R"( + { + "createTime": "123456789", + "task": { + "Name": "task_mock" + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_FALSE(config->Parse()); + + // Type is not of type string + configStr = R"( + { + "createTime": "123456789", + "task": { + "Type": true + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_FALSE(config->Parse()); + + // unsupported Task + configStr = R"( + { + "createTime": "123456789", + "task": { + "Type": "task_unknown" + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_FALSE(config->Parse()); +} + +UNIT_TEST_CASE(TaskConfigUnittest, HandleValidConfig) +UNIT_TEST_CASE(TaskConfigUnittest, HandleInvalidCreateTime) +UNIT_TEST_CASE(TaskConfigUnittest, HandleInvalidTask) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/plugin/PluginMock.h b/core/unittest/plugin/PluginMock.h index 0713b08bcf..dbc9072ed7 100644 --- a/core/unittest/plugin/PluginMock.h +++ b/core/unittest/plugin/PluginMock.h @@ -28,6 +28,8 @@ #include "pipeline/plugin/interface/Input.h" #include "pipeline/plugin/interface/Processor.h" #include "pipeline/queue/SenderQueueManager.h" +#include "task_pipeline/Task.h" +#include "task_pipeline/TaskRegistry.h" namespace logtail { @@ -138,6 +140,25 @@ class FlusherHttpMock : public HttpFlusher { const std::string FlusherHttpMock::sName = "flusher_http_mock"; +class TaskMock : public Task { +public: + static const std::string sName; + + const std::string& Name() const override { return sName; } + bool Init(const Json::Value& config) override { + if (config.isMember("Valid")) { + return config["Valid"].asBool(); + } + return true; + } + void Start() override { mIsRunning = true; } + void Stop(bool isRemoving) { mIsRunning = false; } + + bool mIsRunning = false; +}; + +const std::string TaskMock::sName = "task_mock"; + void LoadPluginMock() { PluginRegistry::GetInstance()->RegisterInputCreator(new StaticInputCreator()); PluginRegistry::GetInstance()->RegisterProcessorCreator(new StaticProcessorCreator()); @@ -146,4 +167,8 @@ void LoadPluginMock() { PluginRegistry::GetInstance()->RegisterFlusherCreator(new StaticFlusherCreator()); } +void LoadTaskMock() { + TaskRegistry::GetInstance()->RegisterCreator(TaskMock::sName, []() { return std::make_unique(); }); +} + } // namespace logtail diff --git a/core/unittest/task_pipeline/CMakeLists.txt b/core/unittest/task_pipeline/CMakeLists.txt new file mode 100644 index 0000000000..b2113a6786 --- /dev/null +++ b/core/unittest/task_pipeline/CMakeLists.txt @@ -0,0 +1,30 @@ +# Copyright 2023 iLogtail Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.22) +project(task_pipeline_unittest) + +add_executable(task_registry_unittest TaskRegistryUnittest.cpp) +target_link_libraries(task_registry_unittest ${UT_BASE_TARGET}) + +add_executable(task_pipeline_unittest TaskPipelineUnittest.cpp) +target_link_libraries(task_pipeline_unittest ${UT_BASE_TARGET}) + +add_executable(task_pipeline_manager_unittest TaskPipelineManagerUnittest.cpp) +target_link_libraries(task_pipeline_manager_unittest ${UT_BASE_TARGET}) + +include(GoogleTest) +gtest_discover_tests(task_registry_unittest) +gtest_discover_tests(task_pipeline_unittest) +gtest_discover_tests(task_pipeline_manager_unittest) diff --git a/core/unittest/task_pipeline/TaskPipelineManagerUnittest.cpp b/core/unittest/task_pipeline/TaskPipelineManagerUnittest.cpp new file mode 100644 index 0000000000..3519c250c0 --- /dev/null +++ b/core/unittest/task_pipeline/TaskPipelineManagerUnittest.cpp @@ -0,0 +1,40 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "task_pipeline/TaskPipelineManager.h" +#include "unittest/Unittest.h" + +using namespace std; + +namespace logtail { + +class TaskPipelineManagerUnittest : public testing::Test { +public: + void TestPipelineManagement() const; +}; + +void TaskPipelineManagerUnittest::TestPipelineManagement() const { + TaskPipelineManager::GetInstance()->mPipelineNameEntityMap["test1"] = make_unique(); + TaskPipelineManager::GetInstance()->mPipelineNameEntityMap["test2"] = make_unique(); + + APSARA_TEST_EQUAL(2U, TaskPipelineManager::GetInstance()->GetAllPipelineNames().size()); + APSARA_TEST_NOT_EQUAL(nullptr, TaskPipelineManager::GetInstance()->FindPipelineByName("test1")); + APSARA_TEST_EQUAL(nullptr, TaskPipelineManager::GetInstance()->FindPipelineByName("test3")); +} + +UNIT_TEST_CASE(TaskPipelineManagerUnittest, TestPipelineManagement) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/task_pipeline/TaskPipelineUnittest.cpp b/core/unittest/task_pipeline/TaskPipelineUnittest.cpp new file mode 100644 index 0000000000..9332784277 --- /dev/null +++ b/core/unittest/task_pipeline/TaskPipelineUnittest.cpp @@ -0,0 +1,127 @@ +// Copyright 2023 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "common/JsonUtil.h" +#include "task_pipeline/TaskPipeline.h" +#include "task_pipeline/TaskRegistry.h" +#include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" + +using namespace std; + +namespace logtail { + +class TaskPipelineUnittest : public ::testing::Test { +public: + void OnSuccessfulInit() const; + void OnFailedInit() const; + void OnUpdate() const; + +protected: + static void SetUpTestCase() { LoadTaskMock(); } + + static void TearDownTestCase() { TaskRegistry::GetInstance()->UnloadPlugins(); } + +private: + const string configName = "test_config"; +}; + +void TaskPipelineUnittest::OnSuccessfulInit() const { + unique_ptr configJson; + string configStr, errorMsg; + unique_ptr config; + unique_ptr task; + + configStr = R"( + { + "createTime": 1234567890, + "task": { + "Type": "task_mock" + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + auto configPtr = configJson.get(); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_TRUE(config->Parse()); + task.reset(new TaskPipeline()); + APSARA_TEST_TRUE(task->Init(std::move(*config))); + APSARA_TEST_EQUAL(configName, task->Name()); + APSARA_TEST_EQUAL(configPtr, &task->GetConfig()); + APSARA_TEST_EQUAL(1234567890U, task->mCreateTime); + APSARA_TEST_NOT_EQUAL(nullptr, task->mPlugin); + APSARA_TEST_EQUAL(TaskMock::sName, task->mPlugin->Name()); +} + +void TaskPipelineUnittest::OnFailedInit() const { + unique_ptr configJson; + string configStr, errorMsg; + unique_ptr config; + unique_ptr task; + + configStr = R"( + { + "createTime": 1234567890, + "task": { + "Type": "task_mock", + "Valid": false + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_TRUE(config->Parse()); + task.reset(new TaskPipeline()); + APSARA_TEST_FALSE(task->Init(std::move(*config))); +} + +void TaskPipelineUnittest::OnUpdate() const { + unique_ptr configJson; + string configStr, errorMsg; + unique_ptr config; + unique_ptr task; + + configStr = R"( + { + "createTime": 1234567890, + "task": { + "Type": "task_mock" + } + } + )"; + configJson.reset(new Json::Value()); + APSARA_TEST_TRUE(ParseJsonTable(configStr, *configJson, errorMsg)); + config.reset(new TaskConfig(configName, std::move(configJson))); + APSARA_TEST_TRUE(config->Parse()); + task.reset(new TaskPipeline()); + APSARA_TEST_TRUE(task->Init(std::move(*config))); + + auto ptr = static_cast(task->mPlugin.get()); + task->Start(); + APSARA_TEST_TRUE(ptr->mIsRunning); + task->Stop(true); + APSARA_TEST_FALSE(ptr->mIsRunning); +} + +UNIT_TEST_CASE(TaskPipelineUnittest, OnSuccessfulInit) +UNIT_TEST_CASE(TaskPipelineUnittest, OnFailedInit) +UNIT_TEST_CASE(TaskPipelineUnittest, OnUpdate) + +} // namespace logtail + +UNIT_TEST_MAIN diff --git a/core/unittest/task_pipeline/TaskRegistryUnittest.cpp b/core/unittest/task_pipeline/TaskRegistryUnittest.cpp new file mode 100644 index 0000000000..05e84a1598 --- /dev/null +++ b/core/unittest/task_pipeline/TaskRegistryUnittest.cpp @@ -0,0 +1,50 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "task_pipeline/TaskRegistry.h" +#include "unittest/Unittest.h" +#include "unittest/plugin/PluginMock.h" + +using namespace std; + +namespace logtail { + +class TaskRegistryUnittest : public testing::Test { +public: + void TestCreateTask() const; + void TestValidPlugin() const; + +protected: + void SetUp() override { LoadTaskMock(); } + void TearDown() override { TaskRegistry::GetInstance()->UnloadPlugins(); } +}; + +void TaskRegistryUnittest::TestCreateTask() const { + auto input = TaskRegistry::GetInstance()->CreateTask(TaskMock::sName); + APSARA_TEST_NOT_EQUAL(nullptr, input); +} + +void TaskRegistryUnittest::TestValidPlugin() const { + APSARA_TEST_TRUE(TaskRegistry::GetInstance()->IsValidPlugin("task_mock")); + APSARA_TEST_FALSE(TaskRegistry::GetInstance()->IsValidPlugin("task_unknown")); +} + +UNIT_TEST_CASE(TaskRegistryUnittest, TestCreateTask) +UNIT_TEST_CASE(TaskRegistryUnittest, TestValidPlugin) + +} // namespace logtail + +UNIT_TEST_MAIN From f5c7c15b41d6f46f2ee6d10db9d9f7acc6b007d8 Mon Sep 17 00:00:00 2001 From: dog Date: Wed, 20 Nov 2024 11:51:31 +0800 Subject: [PATCH 03/11] feat: add input raw event and event pool (#1870) --- core/application/Application.cpp | 2 + .../inner/ProcessorPromParseMetricNative.cpp | 8 ++-- core/prometheus/PrometheusInputRunner.cpp | 7 ++- core/prometheus/PrometheusInputRunner.h | 6 +-- core/prometheus/schedulers/BaseScheduler.cpp | 10 +++++ core/prometheus/schedulers/BaseScheduler.h | 8 +++- .../prometheus/schedulers/ScrapeScheduler.cpp | 10 +---- core/prometheus/schedulers/ScrapeScheduler.h | 15 +++---- .../schedulers/TargetSubscriberScheduler.cpp | 5 +-- .../schedulers/TargetSubscriberScheduler.h | 2 - ...ProcessorPromParseMetricNativeUnittest.cpp | 4 +- .../prometheus/ScrapeSchedulerUnittest.cpp | 45 ++++++++++--------- 12 files changed, 65 insertions(+), 57 deletions(-) diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 08df561cd2..054a424af9 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -48,6 +48,7 @@ #include "pipeline/queue/SenderQueueManager.h" #include "plugin/flusher/sls/DiskBufferWriter.h" #include "plugin/input/InputFeedbackInterfaceRegistry.h" +#include "prometheus/PrometheusInputRunner.h" #include "runner/FlusherRunner.h" #include "runner/ProcessorRunner.h" #include "runner/sink/http/HttpSink.h" @@ -326,6 +327,7 @@ void Application::Start() { // GCOVR_EXCL_START // destruct event handlers here so that it will not block file reading task ConfigManager::GetInstance()->DeleteHandlers(); + PrometheusInputRunner::GetInstance()->CheckGC(); this_thread::sleep_for(chrono::seconds(1)); } diff --git a/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp b/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp index 06de253a4c..95b81a569a 100644 --- a/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp +++ b/core/plugin/processor/inner/ProcessorPromParseMetricNative.cpp @@ -3,10 +3,10 @@ #include #include "common/StringTools.h" -#include "models/LogEvent.h" #include "models/MetricEvent.h" #include "models/PipelineEventGroup.h" #include "models/PipelineEventPtr.h" +#include "models/RawEvent.h" #include "prometheus/Constants.h" using namespace std; @@ -43,7 +43,7 @@ void ProcessorPromParseMetricNative::Process(PipelineEventGroup& eGroup) { } bool ProcessorPromParseMetricNative::IsSupportedEvent(const PipelineEventPtr& e) const { - return e.Is(); + return e.Is(); } bool ProcessorPromParseMetricNative::ProcessEvent(PipelineEventPtr& e, @@ -53,9 +53,9 @@ bool ProcessorPromParseMetricNative::ProcessEvent(PipelineEventPtr& e, if (!IsSupportedEvent(e)) { return false; } - auto& sourceEvent = e.Cast(); + auto& sourceEvent = e.Cast(); std::unique_ptr metricEvent = eGroup.CreateMetricEvent(true); - if (parser.ParseLine(sourceEvent.GetContent(prometheus::PROMETHEUS), *metricEvent)) { + if (parser.ParseLine(sourceEvent.GetContent(), *metricEvent)) { metricEvent->SetTag(string(prometheus::NAME), metricEvent->GetName()); newEvents.emplace_back(std::move(metricEvent), true, nullptr); } diff --git a/core/prometheus/PrometheusInputRunner.cpp b/core/prometheus/PrometheusInputRunner.cpp index 59cc58d4d5..ca49e35091 100644 --- a/core/prometheus/PrometheusInputRunner.cpp +++ b/core/prometheus/PrometheusInputRunner.cpp @@ -47,6 +47,7 @@ PrometheusInputRunner::PrometheusInputRunner() : mServiceHost(STRING_FLAG(loong_collector_operator_service)), mServicePort(INT32_FLAG(loong_collector_operator_service_port)), mPodName(STRING_FLAG(_pod_name_)), + mEventPool(true), mUnRegisterMs(0) { mClient = std::make_unique(); mTimer = std::make_shared(); @@ -83,7 +84,7 @@ void PrometheusInputRunner::UpdateScrapeInput(std::shared_ptrInitSelfMonitor(defaultLabels); targetSubscriber->mUnRegisterMs = mUnRegisterMs.load(); - targetSubscriber->SetTimer(mTimer); + targetSubscriber->SetComponent(mTimer, &mEventPool); auto randSleepMilliSec = GetRandSleepMilliSec( targetSubscriber->GetId(), prometheus::RefeshIntervalSeconds, GetCurrentTimeInMilliSeconds()); auto firstExecTime = std::chrono::steady_clock::now() + std::chrono::milliseconds(randSleepMilliSec); @@ -294,4 +295,8 @@ string PrometheusInputRunner::GetAllProjects() { } return result; } + +void PrometheusInputRunner::CheckGC() { + mEventPool.CheckGC(); +} }; // namespace logtail \ No newline at end of file diff --git a/core/prometheus/PrometheusInputRunner.h b/core/prometheus/PrometheusInputRunner.h index 89ae961ce1..2823d562a4 100644 --- a/core/prometheus/PrometheusInputRunner.h +++ b/core/prometheus/PrometheusInputRunner.h @@ -22,7 +22,6 @@ #include "common/Lock.h" #include "common/timer/Timer.h" -#include "monitor/MetricManager.h" #include "monitor/MetricTypes.h" #include "prometheus/schedulers/TargetSubscriberScheduler.h" #include "runner/InputRunner.h" @@ -42,6 +41,7 @@ class PrometheusInputRunner : public InputRunner { static PrometheusInputRunner sInstance; return &sInstance; } + void CheckGC(); // input plugin update void UpdateScrapeInput(std::shared_ptr targetSubscriber, @@ -70,13 +70,13 @@ class PrometheusInputRunner : public InputRunner { std::atomic mIsThreadRunning = true; std::future mThreadRes; - std::unique_ptr mClient; - std::string mServiceHost; int32_t mServicePort; std::string mPodName; + std::unique_ptr mClient; std::shared_ptr mTimer; + EventPool mEventPool; mutable ReadWriteLock mSubscriberMapRWLock; std::map> mTargetSubscriberSchedulerMap; diff --git a/core/prometheus/schedulers/BaseScheduler.cpp b/core/prometheus/schedulers/BaseScheduler.cpp index db7de4ae79..af564f1622 100644 --- a/core/prometheus/schedulers/BaseScheduler.cpp +++ b/core/prometheus/schedulers/BaseScheduler.cpp @@ -1,5 +1,10 @@ #include "prometheus/schedulers/BaseScheduler.h" +#include "common/timer/Timer.h" +#include "models/EventPool.h" + +using namespace std; + namespace logtail { void BaseScheduler::ExecDone() { mExecCount++; @@ -28,4 +33,9 @@ bool BaseScheduler::IsCancelled() { ReadLock lock(mLock); return !mValidState; } + +void BaseScheduler::SetComponent(shared_ptr timer, EventPool* eventPool) { + mTimer = std::move(timer); + mEventPool = eventPool; +} } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/BaseScheduler.h b/core/prometheus/schedulers/BaseScheduler.h index e66e612d78..26739cdcd0 100644 --- a/core/prometheus/schedulers/BaseScheduler.h +++ b/core/prometheus/schedulers/BaseScheduler.h @@ -4,6 +4,8 @@ #include #include "common/http/HttpResponse.h" +#include "common/timer/Timer.h" +#include "models/EventPool.h" #include "prometheus/async/PromFuture.h" namespace logtail { @@ -20,9 +22,10 @@ class BaseScheduler { void SetFirstExecTime(std::chrono::steady_clock::time_point firstExecTime); void DelayExecTime(uint64_t delaySeconds); - virtual void Cancel(); + void SetComponent(std::shared_ptr timer, EventPool* eventPool); + protected: bool IsCancelled(); @@ -35,5 +38,8 @@ class BaseScheduler { bool mValidState = true; std::shared_ptr> mFuture; std::shared_ptr> mIsContextValidFuture; + + std::shared_ptr mTimer; + EventPool* mEventPool = nullptr; }; } // namespace logtail \ No newline at end of file diff --git a/core/prometheus/schedulers/ScrapeScheduler.cpp b/core/prometheus/schedulers/ScrapeScheduler.cpp index a830558227..02fca2caec 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.cpp +++ b/core/prometheus/schedulers/ScrapeScheduler.cpp @@ -24,13 +24,11 @@ #include "common/StringTools.h" #include "common/TimeUtil.h" #include "common/timer/HttpRequestTimerEvent.h" -#include "common/timer/Timer.h" #include "logger/Logger.h" #include "pipeline/queue/ProcessQueueItem.h" #include "pipeline/queue/ProcessQueueManager.h" #include "pipeline/queue/QueueKey.h" #include "prometheus/Constants.h" -#include "prometheus/Utils.h" #include "prometheus/async/PromFuture.h" #include "prometheus/async/PromHttpRequest.h" #include "sdk/Common.h" @@ -86,8 +84,6 @@ ScrapeScheduler::ScrapeScheduler(std::shared_ptr scrapeConfigPtr, mHash = mScrapeConfigPtr->mJobName + tmpTargetURL + ToString(mTargetLabels.Hash()); mInstance = mHost + ":" + ToString(mPort); mInterval = mScrapeConfigPtr->mScrapeIntervalSeconds; - - mParser = make_unique(); } void ScrapeScheduler::OnMetricResult(HttpResponse& response, uint64_t timestampMilliSec) { @@ -214,7 +210,7 @@ std::unique_ptr ScrapeScheduler::BuildScrapeTimerEvent(std::chrono:: mScrapeConfigPtr->mRequestHeaders, "", HttpResponse( - new PromMetricResponseBody(), + new PromMetricResponseBody(mEventPool), [](void* ptr) { delete static_cast(ptr); }, PromMetricWriteCallback), mScrapeConfigPtr->mScrapeTimeoutSeconds, @@ -238,10 +234,6 @@ void ScrapeScheduler::Cancel() { } } -void ScrapeScheduler::SetTimer(std::shared_ptr timer) { - mTimer = std::move(timer); -} - void ScrapeScheduler::InitSelfMonitor(const MetricLabels& defaultLabels) { mSelfMonitor = std::make_shared(); MetricLabels labels = defaultLabels; diff --git a/core/prometheus/schedulers/ScrapeScheduler.h b/core/prometheus/schedulers/ScrapeScheduler.h index 6a606627f2..00ac2d989a 100644 --- a/core/prometheus/schedulers/ScrapeScheduler.h +++ b/core/prometheus/schedulers/ScrapeScheduler.h @@ -21,14 +21,11 @@ #include "BaseScheduler.h" #include "common/http/HttpResponse.h" -#include "common/timer/Timer.h" #include "models/PipelineEventGroup.h" #include "monitor/MetricTypes.h" #include "pipeline/queue/QueueKey.h" -#include "prometheus/Constants.h" #include "prometheus/PromSelfMonitor.h" #include "prometheus/Utils.h" -#include "prometheus/labels/TextParser.h" #include "prometheus/schedulers/ScrapeConfig.h" #ifdef APSARA_UNIT_TEST_MAIN @@ -43,13 +40,15 @@ struct PromMetricResponseBody { PipelineEventGroup mEventGroup; std::string mCache; size_t mRawSize = 0; + EventPool* mEventPool = nullptr; - PromMetricResponseBody() : mEventGroup(std::make_shared()) {}; + explicit PromMetricResponseBody(EventPool* eventPool) + : mEventGroup(std::make_shared()), mEventPool(eventPool) {}; void AddEvent(char* line, size_t len) { if (IsValidMetric(StringView(line, len))) { - auto* e = mEventGroup.AddLogEvent(); + auto* e = mEventGroup.AddRawEvent(true, mEventPool); auto sb = mEventGroup.GetSourceBuffer()->CopyString(line, len); - e->SetContentNoCopy(prometheus::PROMETHEUS, StringView(sb.data, sb.size)); + e->SetContentNoCopy(sb); } } void FlushCache() { @@ -70,7 +69,6 @@ class ScrapeScheduler : public BaseScheduler { ~ScrapeScheduler() override = default; void OnMetricResult(HttpResponse&, uint64_t timestampMilliSec); - void SetTimer(std::shared_ptr timer); std::string GetId() const; @@ -94,11 +92,8 @@ class ScrapeScheduler : public BaseScheduler { std::string mInstance; Labels mTargetLabels; - std::unique_ptr mParser; - QueueKey mQueueKey; size_t mInputIndex; - std::shared_ptr mTimer; // auto metrics uint64_t mScrapeTimestampMilliSec = 0; diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp index 178c952dba..42381f1229 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.cpp +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.cpp @@ -223,7 +223,7 @@ TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGr auto scrapeScheduler = std::make_shared(mScrapeConfigPtr, host, port, resultLabel, mQueueKey, mInputIndex); - scrapeScheduler->SetTimer(mTimer); + scrapeScheduler->SetComponent(mTimer, mEventPool); auto randSleepMilliSec = GetRandSleepMilliSec( scrapeScheduler->GetId(), mScrapeConfigPtr->mScrapeIntervalSeconds, GetCurrentTimeInMilliSeconds()); @@ -236,9 +236,6 @@ TargetSubscriberScheduler::BuildScrapeSchedulerSet(std::vector& targetGr return scrapeSchedulerMap; } -void TargetSubscriberScheduler::SetTimer(shared_ptr timer) { - mTimer = std::move(timer); -} string TargetSubscriberScheduler::GetId() const { return mJobName; diff --git a/core/prometheus/schedulers/TargetSubscriberScheduler.h b/core/prometheus/schedulers/TargetSubscriberScheduler.h index f6dc0af7aa..af5e1493fd 100644 --- a/core/prometheus/schedulers/TargetSubscriberScheduler.h +++ b/core/prometheus/schedulers/TargetSubscriberScheduler.h @@ -41,7 +41,6 @@ class TargetSubscriberScheduler : public BaseScheduler { bool operator<(const TargetSubscriberScheduler& other) const; void OnSubscription(HttpResponse&, uint64_t); - void SetTimer(std::shared_ptr timer); void SubscribeOnce(std::chrono::steady_clock::time_point execTime); std::string GetId() const; @@ -79,7 +78,6 @@ class TargetSubscriberScheduler : public BaseScheduler { std::unordered_map> mScrapeSchedulerMap; std::string mJobName; - std::shared_ptr mTimer; std::string mETag; diff --git a/core/unittest/processor/ProcessorPromParseMetricNativeUnittest.cpp b/core/unittest/processor/ProcessorPromParseMetricNativeUnittest.cpp index 62bf5e3dc2..c0481a1b94 100644 --- a/core/unittest/processor/ProcessorPromParseMetricNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorPromParseMetricNativeUnittest.cpp @@ -85,8 +85,8 @@ void ProcessorParsePrometheusMetricUnittest::TestProcess() { if (newLine.empty() || newLine[0] == '#') { continue; } - auto* MetricEvent = eGroup.AddLogEvent(); - MetricEvent->SetContent(prometheus::PROMETHEUS, newLine); + auto* metricEvent = eGroup.AddRawEvent(); + metricEvent->SetContent(newLine); } return eGroup; diff --git a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp index d51cc4e57e..2c5138768b 100644 --- a/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp +++ b/core/unittest/prometheus/ScrapeSchedulerUnittest.cpp @@ -21,6 +21,7 @@ #include "common/StringTools.h" #include "common/http/HttpResponse.h" #include "common/timer/Timer.h" +#include "models/RawEvent.h" #include "prometheus/Constants.h" #include "prometheus/async/PromFuture.h" #include "prometheus/labels/Labels.h" @@ -65,8 +66,11 @@ void ScrapeSchedulerUnittest::TestInitscrapeScheduler() { } void ScrapeSchedulerUnittest::TestProcess() { + EventPool eventPool{true}; HttpResponse httpResponse = HttpResponse( - new PromMetricResponseBody(), [](void* ptr) { delete static_cast(ptr); }, PromMetricWriteCallback); + new PromMetricResponseBody(&eventPool), + [](void* ptr) { delete static_cast(ptr); }, + PromMetricWriteCallback); Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); @@ -112,8 +116,11 @@ void ScrapeSchedulerUnittest::TestProcess() { } void ScrapeSchedulerUnittest::TestStreamMetricWriteCallback() { + EventPool eventPool{true}; HttpResponse httpResponse = HttpResponse( - new PromMetricResponseBody(), [](void* ptr) { delete static_cast(ptr); }, PromMetricWriteCallback); + new PromMetricResponseBody(&eventPool), + [](void* ptr) { delete static_cast(ptr); }, + PromMetricWriteCallback); Labels labels; labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); @@ -148,33 +155,27 @@ void ScrapeSchedulerUnittest::TestStreamMetricWriteCallback() { auto& res = httpResponse.GetBody()->mEventGroup; APSARA_TEST_EQUAL(7UL, res.GetEvents().size()); APSARA_TEST_EQUAL("go_gc_duration_seconds{quantile=\"0\"} 1.5531e-05", - res.GetEvents()[0].Cast().GetContent(prometheus::PROMETHEUS).to_string()); + res.GetEvents()[0].Cast().GetContent()); APSARA_TEST_EQUAL("go_gc_duration_seconds{quantile=\"0.25\"} 3.9357e-05", - res.GetEvents()[1].Cast().GetContent(prometheus::PROMETHEUS).to_string()); + res.GetEvents()[1].Cast().GetContent()); APSARA_TEST_EQUAL("go_gc_duration_seconds{quantile=\"0.5\"} 4.1114e-05", - res.GetEvents()[2].Cast().GetContent(prometheus::PROMETHEUS).to_string()); + res.GetEvents()[2].Cast().GetContent()); APSARA_TEST_EQUAL("go_gc_duration_seconds{quantile=\"0.75\"} 4.3372e-05", - res.GetEvents()[3].Cast().GetContent(prometheus::PROMETHEUS).to_string()); + res.GetEvents()[3].Cast().GetContent()); APSARA_TEST_EQUAL("go_gc_duration_seconds{quantile=\"1\"} 0.000112326", - res.GetEvents()[4].Cast().GetContent(prometheus::PROMETHEUS).to_string()); - APSARA_TEST_EQUAL("go_gc_duration_seconds_sum 0.034885631", - res.GetEvents()[5].Cast().GetContent(prometheus::PROMETHEUS).to_string()); - APSARA_TEST_EQUAL("go_gc_duration_seconds_count 850", - res.GetEvents()[6].Cast().GetContent(prometheus::PROMETHEUS).to_string()); + res.GetEvents()[4].Cast().GetContent()); + APSARA_TEST_EQUAL("go_gc_duration_seconds_sum 0.034885631", res.GetEvents()[5].Cast().GetContent()); + APSARA_TEST_EQUAL("go_gc_duration_seconds_count 850", res.GetEvents()[6].Cast().GetContent()); // httpResponse.GetBody()->mEventGroup = PipelineEventGroup(std::make_shared()); PromMetricWriteCallback( body2.data(), (size_t)1, (size_t)body2.length(), (void*)httpResponse.GetBody()); httpResponse.GetBody()->FlushCache(); APSARA_TEST_EQUAL(11UL, res.GetEvents().size()); - APSARA_TEST_EQUAL("go_goroutines 7", - res.GetEvents()[7].Cast().GetContent(prometheus::PROMETHEUS).to_string()); - APSARA_TEST_EQUAL("go_info{version=\"go1.22.3\"} 1", - res.GetEvents()[8].Cast().GetContent(prometheus::PROMETHEUS).to_string()); - APSARA_TEST_EQUAL("go_memstats_alloc_bytes 6.742688e+06", - res.GetEvents()[9].Cast().GetContent(prometheus::PROMETHEUS).to_string()); - APSARA_TEST_EQUAL("go_memstats_alloc_bytes_total 1.5159292e+08", - res.GetEvents()[10].Cast().GetContent(prometheus::PROMETHEUS).to_string()); + APSARA_TEST_EQUAL("go_goroutines 7", res.GetEvents()[7].Cast().GetContent()); + APSARA_TEST_EQUAL("go_info{version=\"go1.22.3\"} 1", res.GetEvents()[8].Cast().GetContent()); + APSARA_TEST_EQUAL("go_memstats_alloc_bytes 6.742688e+06", res.GetEvents()[9].Cast().GetContent()); + APSARA_TEST_EQUAL("go_memstats_alloc_bytes_total 1.5159292e+08", res.GetEvents()[10].Cast().GetContent()); } void ScrapeSchedulerUnittest::TestReceiveMessage() { @@ -197,7 +198,8 @@ void ScrapeSchedulerUnittest::TestScheduler() { labels.Set(prometheus::ADDRESS_LABEL_NAME, "localhost:8080"); ScrapeScheduler event(mScrapeConfig, "localhost", 8080, labels, 0, 0); auto timer = make_shared(); - event.SetTimer(timer); + EventPool eventPool{true}; + event.SetComponent(timer, &eventPool); event.ScheduleNext(); APSARA_TEST_TRUE(timer->mQueue.size() == 1); @@ -215,7 +217,8 @@ void ScrapeSchedulerUnittest::TestQueueIsFull() { auto defaultLabels = MetricLabels(); event.InitSelfMonitor(defaultLabels); auto timer = make_shared(); - event.SetTimer(timer); + EventPool eventPool{true}; + event.SetComponent(timer, &eventPool); auto now = std::chrono::steady_clock::now(); event.SetFirstExecTime(now); event.ScheduleNext(); From 7342a226940288122ca45fed0c16ecd506c4dfea Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Wed, 20 Nov 2024 14:34:58 +0800 Subject: [PATCH 04/11] The plugin log name and checkpoint name support the logtail mode. (#1901) --- core/app_config/AppConfig.cpp | 42 +++- core/app_config/AppConfig.h | 5 + core/application/Application.cpp | 7 +- core/common/LogtailCommonFlags.cpp | 2 +- core/go_pipeline/LogtailPlugin.cpp | 3 + core/logtail.cpp | 7 + core/monitor/Monitor.cpp | 2 +- core/pipeline/Pipeline.cpp | 5 +- docs/cn/SUMMARY.md | 2 + docs/cn/installation/logtail-mode.md | 79 ++++++++ docs/cn/installation/loongcollector-dir.md | 225 +++++++++++++++++++++ pkg/config/global_config.go | 33 +-- pkg/logger/logger.go | 6 +- pkg/logger/logger_test.go | 4 +- pkg/logtail/libGoPluginAdapter.so | Bin 36000 -> 36000 bytes pluginmanager/checkpoint_manager.go | 8 +- pluginmanager/plugin_manager.go | 4 +- 17 files changed, 405 insertions(+), 29 deletions(-) create mode 100644 docs/cn/installation/logtail-mode.md create mode 100644 docs/cn/installation/loongcollector-dir.md diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index 976a7b3339..e9ecf50e61 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -491,6 +491,46 @@ string GetPipelineConfigDir() { } } +string GetPluginLogName() { + if (BOOL_FLAG(logtail_mode)) { + return "logtail_plugin.LOG"; + } else { + return "go_plugin.LOG"; + } +} + +std::string GetVersionTag() { + if (BOOL_FLAG(logtail_mode)) { + return "logtail_version"; + } else { + return "loongcollector_version"; + } +} + +std::string GetGoPluginCheckpoint() { + if (BOOL_FLAG(logtail_mode)) { + return "checkpoint"; + } else { + return "go_plugin_checkpoint"; + } +} + +std::string GetAgentName() { + if (BOOL_FLAG(logtail_mode)) { + return "ilogtail"; + } else { + return "loongcollector"; + } +} + +std::string GetMonitorInfoFileName() { + if (BOOL_FLAG(logtail_mode)) { + return "logtail_monitor_info"; + } else { + return "loongcollector_monitor_info"; + } +} + AppConfig::AppConfig() { LOG_INFO(sLogger, ("AppConfig AppConfig", "success")); SetIlogtailConfigJson(""); @@ -1646,7 +1686,7 @@ void AppConfig::SetLoongcollectorConfDir(const std::string& dirPath) { // = AbsolutePath(STRING_FLAG(ilogtail_local_yaml_config_dir), mLogtailSysConfDir) + PATH_SEPARATOR; // mUserRemoteYamlConfigDirPath // = AbsolutePath(STRING_FLAG(ilogtail_remote_yaml_config_dir), mLogtailSysConfDir) + PATH_SEPARATOR; - LOG_INFO(sLogger, ("set loongcollector conf dir", mLoongcollectorConfDir)); + LOG_INFO(sLogger, ("set " + GetAgentName() + " conf dir", mLoongcollectorConfDir)); } bool AppConfig::IsHostPathMatchBlacklist(const string& dirPath) const { diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 4a28cf7c1a..e2f9929c38 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -55,6 +55,11 @@ std::string GetSendBufferFileNamePrefix(); std::string GetLegacyUserLocalConfigFilePath(); std::string GetExactlyOnceCheckpoint(); std::string GetPipelineConfigDir(); +std::string GetPluginLogName(); +std::string GetVersionTag(); +std::string GetGoPluginCheckpoint(); +std::string GetAgentName(); +std::string GetMonitorInfoFileName(); template class DoubleBuffer { diff --git a/core/application/Application.cpp b/core/application/Application.cpp index 054a424af9..d6abc71d96 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -32,8 +32,8 @@ #include "common/version.h" #include "config/ConfigDiff.h" #include "config/InstanceConfigManager.h" -#include "config/watcher/PipelineConfigWatcher.h" #include "config/watcher/InstanceConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" #include "file_server/ConfigManager.h" #include "file_server/EventDispatcher.h" #include "file_server/FileServer.h" @@ -175,9 +175,10 @@ void Application::Init() { appInfoJson["UUID"] = Json::Value(Application::GetInstance()->GetUUID()); appInfoJson["instance_id"] = Json::Value(Application::GetInstance()->GetInstanceId()); #ifdef __ENTERPRISE__ - appInfoJson["loongcollector_version"] = Json::Value(ILOGTAIL_VERSION); + appInfoJson["host_id"] = Json::Value(FetchHostId()); + appInfoJson[GetVersionTag()] = Json::Value(ILOGTAIL_VERSION); #else - appInfoJson["loongcollector_version"] = Json::Value(string(ILOGTAIL_VERSION) + " Community Edition"); + appInfoJson[GetVersionTag()] = Json::Value(string(ILOGTAIL_VERSION) + " Community Edition"); appInfoJson["git_hash"] = Json::Value(ILOGTAIL_GIT_HASH); appInfoJson["build_date"] = Json::Value(ILOGTAIL_BUILD_DATE); #endif diff --git a/core/common/LogtailCommonFlags.cpp b/core/common/LogtailCommonFlags.cpp index b2cea76fda..1d70138f27 100644 --- a/core/common/LogtailCommonFlags.cpp +++ b/core/common/LogtailCommonFlags.cpp @@ -81,7 +81,7 @@ // app config DEFINE_FLAG_STRING(ilogtail_config, "set dataserver & configserver address; (optional)set cpu,mem,bufflerfile,buffermap and etc.", - "loongcollector_config.json"); + "ilogtail_config.json"); DEFINE_FLAG_BOOL(enable_full_drain_mode, "", false); DEFINE_FLAG_INT32(cpu_limit_num, "cpu violate limit num before shutdown", 10); DEFINE_FLAG_INT32(mem_limit_num, "memory violate limit num before shutdown", 10); diff --git a/core/go_pipeline/LogtailPlugin.cpp b/core/go_pipeline/LogtailPlugin.cpp index 9ba4f91eb0..6e41963c82 100644 --- a/core/go_pipeline/LogtailPlugin.cpp +++ b/core/go_pipeline/LogtailPlugin.cpp @@ -65,6 +65,9 @@ LogtailPlugin::LogtailPlugin() { mPluginCfg["LoongcollectorConfDir"] = AppConfig::GetInstance()->GetLoongcollectorConfDir(); mPluginCfg["LoongcollectorLogDir"] = GetAgentLogDir(); mPluginCfg["LoongcollectorDataDir"] = GetAgentDataDir(); + mPluginCfg["LoongcollectorPluginLogName"] = GetPluginLogName(); + mPluginCfg["LoongcollectorVersionTag"] = GetVersionTag(); + mPluginCfg["LoongcollectorCheckPointFile"] = GetGoPluginCheckpoint(); mPluginCfg["LoongcollectorThirdPartyDir"] = GetAgentThirdPartyDir(); mPluginCfg["HostIP"] = LoongCollectorMonitor::mIpAddr; mPluginCfg["Hostname"] = LoongCollectorMonitor::mHostname; diff --git a/core/logtail.cpp b/core/logtail.cpp index 52e7e7db9c..90a2d187e9 100644 --- a/core/logtail.cpp +++ b/core/logtail.cpp @@ -47,6 +47,7 @@ DECLARE_FLAG_INT32(data_server_port); DECLARE_FLAG_BOOL(enable_env_ref_in_config); DECLARE_FLAG_BOOL(enable_sls_metrics_format); DECLARE_FLAG_BOOL(enable_containerd_upper_dir_detect); +DECLARE_FLAG_BOOL(logtail_mode); void HandleSigtermSignal(int signum, siginfo_t* info, void* context) { LOG_INFO(sLogger, ("received signal", "SIGTERM")); @@ -74,6 +75,12 @@ void enable_core(void) { static void overwrite_community_edition_flags() { // support run in installation dir on default + if(BOOL_FLAG(logtail_mode)) { + STRING_FLAG(logtail_sys_conf_dir) = "."; + STRING_FLAG(check_point_filename) = "checkpoint/logtail_check_point"; + STRING_FLAG(default_buffer_file_path) = "checkpoint"; + STRING_FLAG(ilogtail_docker_file_path_config) = "checkpoint/docker_path_config.json"; + } STRING_FLAG(metrics_report_method) = ""; INT32_FLAG(data_server_port) = 443; BOOL_FLAG(enable_env_ref_in_config) = true; diff --git a/core/monitor/Monitor.cpp b/core/monitor/Monitor.cpp index f83f9f37aa..d253cb750f 100644 --- a/core/monitor/Monitor.cpp +++ b/core/monitor/Monitor.cpp @@ -489,7 +489,7 @@ void LogtailMonitor::DumpToLocal(const sls_logs::LogGroup& logGroup) { } bool LogtailMonitor::DumpMonitorInfo(time_t monitorTime) { - string path = GetAgentLogDir() + "loongcollector_monitor_info"; + string path = GetAgentLogDir() + GetMonitorInfoFileName(); ofstream outfile(path.c_str(), ofstream::app); if (!outfile) return false; diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 76e49682fc..359385c6c5 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -20,6 +20,7 @@ #include #include +#include "app_config/AppConfig.h" #include "common/Flags.h" #include "common/ParamExtractor.h" #include "go_pipeline/LogtailPlugin.h" @@ -497,7 +498,7 @@ bool Pipeline::LoadGoPipelines() const { mContext.GetRegion(), mContext.GetLogstoreKey())) { LOG_ERROR(mContext.GetLogger(), - ("failed to init pipeline", "Go pipeline is invalid, see go_plugin.LOG for detail")( + ("failed to init pipeline", "Go pipeline is invalid, see " + GetPluginLogName() + " for detail")( "Go pipeline num", "2")("Go pipeline content", content)("config", mName)); AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM, "Go pipeline is invalid, content: " + content + ", config: " + mName, @@ -516,7 +517,7 @@ bool Pipeline::LoadGoPipelines() const { mContext.GetRegion(), mContext.GetLogstoreKey())) { LOG_ERROR(mContext.GetLogger(), - ("failed to init pipeline", "Go pipeline is invalid, see go_plugin.LOG for detail")( + ("failed to init pipeline", "Go pipeline is invalid, see " + GetPluginLogName() + " for detail")( "Go pipeline num", "1")("Go pipeline content", content)("config", mName)); AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM, "Go pipeline is invalid, content: " + content + ", config: " + mName, diff --git a/docs/cn/SUMMARY.md b/docs/cn/SUMMARY.md index 037863731a..f830edbb95 100644 --- a/docs/cn/SUMMARY.md +++ b/docs/cn/SUMMARY.md @@ -30,6 +30,8 @@ * [Docker镜像](installation/sources/docker-image.md) * [编译依赖](installation/sources/dependencies.md) * [镜像站](installation/mirrors.md) +* [LoongCollector 的 目录结构说明](installation/loongcollector-dir.md) +* [LoongCollector 的 Logtail 兼容模式使用指南](installation/logtail-mode.md) ## 概念 diff --git a/docs/cn/installation/logtail-mode.md b/docs/cn/installation/logtail-mode.md new file mode 100644 index 0000000000..bbc6fa7683 --- /dev/null +++ b/docs/cn/installation/logtail-mode.md @@ -0,0 +1,79 @@ +# LoongCollector 的 Logtail 兼容模式使用指南 + +LoongCollector 提供了 Logtail 兼容模式，可以让您在升级到 LoongCollector 后继续使用原有的 Logtail 配置和数据，实现平滑迁移。本文将详细介绍如何配置和使用这个兼容模式。 + +> 在开始之前，请先了解 [LoongCollector 的目录结构说明](loongcollector-dir.md)。 + +## 为什么需要兼容模式？ + +由于 LoongCollector 采用了新的目录结构和配置体系，与原有 Logtail 存在差异，如果您相关的目录文件升级迁移困难，可以选择使用 Logtail 兼容模式。启用兼容模式后，LoongCollector 将： + +- 保持与 Logtail 相同的目录结构 + +- 继续使用 Logtail 的自定义目录配置方式 + +- 继续使用 Logtail 的文件命名格式 + +## 配置方法 + +### 1. 主机环境配置 + +您可以通过以下两种方式之一启用兼容模式： + +**方式一：命令行参数** + +```bash +./loongcollector --logtail_mode=true +``` + +**方式二：环境变量** + +```bash +export logtail_mode=true +./loongcollector +``` + +### 2. 容器环境配置 + +此前的 Logtail 容器镜像中，Logtail 运行时目录为 `/usr/local/ilogtail`，而 LoongCollector 运行时目录为 `/usr/local/loongcollector`。 + +因此，在容器环境中，除了启用兼容模式外，还需要调整目录映射。请按照以下步骤操作： + +1. 需要给LoongCollector容器添加环境变量： + +```bash +logtail_mode=true +``` + +2. 需要调整LoongCollector挂载路径映射： + +将所有 `/usr/local/ilogtail` 路径替换为 `/usr/local/loongcollector`： + +```plaintext +# 常用目录映射示例 +数据检查点： +/usr/local/ilogtail/checkpoint → /usr/local/loongcollector/checkpoint + +采集配置目录： +/usr/local/ilogtail/config → /usr/local/loongcollector/config +``` + +3. 修改容器镜像地址为LoongCollector镜像地址 + +`sls-opensource-registry-vpc.cn-shanghai.cr.aliyuncs.com/loongcollector-community-edition/loongcollector:版本号` + +## 迁移建议 + +为确保平稳迁移，我们建议您： + +1. 先在测试环境进行充分验证 + +2. 选择业务低峰期进行升级 + +3. 做好配置和数据的备份 + +4. 逐步迁移，避免一次性升级所有实例 + +5. 密切监控日志采集状态 + +> **注意**: 迁移过程中请确保数据完整性,建议先在测试环境中进行测试，并非高峰期进行升级操作。 \ No newline at end of file diff --git a/docs/cn/installation/loongcollector-dir.md b/docs/cn/installation/loongcollector-dir.md new file mode 100644 index 0000000000..36d274d626 --- /dev/null +++ b/docs/cn/installation/loongcollector-dir.md @@ -0,0 +1,225 @@ +# LoongCollector 的 目录结构说明 + +## 产品更名说明 + +作为 2024 年产品规划的重要组成部分，iLogtail 产品将正式更名为 LoongCollector。此次更新主要包含以下变更： + +- 程序二进制文件由 iLogtail 更名为 LoongCollector + +- 全面优化目录结构和配置文件布局，提供更清晰的组织方式 + +## 新版目录结构 + +LoongCollector 采用模块化的分层目录设计，以下展示了安装在 /opt/loongcollector 下的标准目录结构： + +库文件： + +- `/opt/loongcollector/libPluginAdapter.so` + +- `/opt/loongcollector/libPluginBase.so` + +自带证书：`/opt/loongcollector/ca-bundle.crt` + +**配置文件目录：**`/opt/loongcollector/conf` + +日志配置文件： + +- `/opt/loongcollector/conf/apsara_log_conf.json` + +- `/opt/loongcollector/conf/plugin_logger.xml` + +标识配置文件： + +- `/opt/loongcollector/conf/user_defined_id` + +采集配置文件：`/opt/loongcollector/conf/continuous_pipeline_config` + +进程级文件：`/opt/loongcollector/conf/instance_config` + +**数据目录：**`/opt/loongcollector/data` + +检查点： + +- `/opt/loongcollector/data/go_plugin_checkpoint` + +- `/opt/loongcollector/data/exactly_once_checkpoint` + +- `/opt/loongcollector/data/file_check_point` + +容器路径映射：`/opt/loongcollector/data/docker_path_config.json` + +未发送数据：`/opt/loongcollector/data/send_buffer_file_xxxxxxxxxxxx` + +Crash临时文件：`/opt/loongcollector/data/backtrace.dat` + +**日志目录：**`/opt/loongcollector/log` + +主要日志：`/opt/loongcollector/log/loongcollector.log` + +Go插件日志：`/opt/loongcollector/log/go_plugin.log` + +日志库初始化日志：`/opt/loongcollector/log/logger_initialization.log` + +Profile日志：`/opt/loongcollector/log/snapshot` + +**run目录：**`/opt/loongcollector/run` + +Pid文件：`/opt/loongcollector/run/loongcollector.pid` + +inotify日志：`/opt/loongcollector/run/inotify_watcher_dirs` + +进程信息日志：`/opt/loongcollector/run/app_info.json` + +```plaintext +/ +└── /opt/loongcollector/ + ├── loongcollector # 主程序 + ├── libPluginAdapter.so + ├── libPluginBase.so + ├── ca-bundle.crt + ├── plugins/ # 插件目录 + │ └── custom plugins # 自定义插件 + ├── dump # 仅由 service_http_server 输入插件使用 + ├── thirdparty/ # 第三方依赖 + │ ├── jvm + │ └── telegraf + ├── conf/ # 配置目录 + │ ├── scripts + │ ├── apsara_log_conf.json + │ ├── plugin_logger.xml + │ ├── user_defined_id + │ ├── authorization.json + │ ├── continuous_pipeline_config/ + │ │ ├── local/ + │ │ │ └── collect_stdout.json + │ │ └── remote/ + │ │ └── collect_file.json + │ └── instance_config/ + │ ├── local/ + │ │ ├── loongcollector_config.json（loongcollector配置） + │ │ └── ebpf.json + │ └── remote/ + │ ├── region.json + │ └── resource.json + ├── data/ # 数据目录 + │ ├── file_check_point # 文件采集的checkpoint + │ ├── exactly_once_checkpoint/ + │ ├── go_plugin_checkpoint/ # go插件采集的checkpoint + │ ├── docker_path_config.json + │ ├── send_buffer_file_xxxxxxxxxxxx + │ └── backtrace.dat + ├── log/ # 日志目录 + │ ├── loongcollector.log + │ ├── loongcollector.log.1 + │ ├── go_plugin.log + │ ├── go_plugin.log.1 + │ ├── logger_initialization.log + │ └── snapshot/ + └── run/ + ├── loongcollector.pid + ├── inotify_watcher_dirs + └── app_info.json +``` + +## 目录自定义配置 + +### 支持的自定义目录参数 + +LoongCollector 提供以下参数用于自定义各类目录位置： + +- `loongcollector_conf_dir`: 配置目录 + +- `loongcollector_log_dir`: 日志目录 + +- `loongcollector_data_dir`: 数据目录 + +- `loongcollector_run_dir`: 运行时目录 + +- `loongcollector_third_party_dir`: 第三方依赖目录 + +### 配置方式 + +1. 命令行参数: + +```bash +./loongcollector --loongcollector_conf_dir=/custom/path/conf +``` + +2. 环境变量: + +```bash +export loongcollector_conf_dir=/custom/path/conf +./loongcollector +``` + +## 命名变更对照表 + +为确保命名一致性，我们对以下文件和目录进行了规范化命名： + +| 文件/目录作用 | 原命名 | 新命名 | +| ------------------------ | ----------------------- | --------------------------- | +| agent可观测文件 | logtail_monitor_info | loongcollector_monitor_info | +| go插件采集的checkpoint | checkpoint | go_plugin_checkpoint | +| go插件运行日志 | logtail_plugin.LOG | go_plugin.LOG | +| 采集配置目录名 | config | continuous_pipeline_config | +| exactly_once的checkpoint | checkpoint_v2 | exactly_once_checkpoint | +| agent的发送缓冲buffer文件 | logtail_buffer_file_xxx | send_buffer_file_xxx | +| agent可观测文件 | ilogtail_status.LOG | loongcollector_status.LOG | +| agent运行日志 | ilogtail.LOG | loongcollector.LOG | + +## 配置兼容性说明 + +为简化配置体系，以下原 Logtail 配置项将不再默认支持： + +- sls_observer_ebpf_host_path + +- logtail_snapshot_dir + +- inotify_watcher_dirs_dump_filename + +- local_event_data_file_name + +- crash_stack_file_name + +- check_point_filename + +- adhoc_check_point_file_dir + +- app_info_file + +- ilogtail_config + +- ilogtail_config_env_name + +- logtail_sys_conf_dir + +- ALIYUN_LOGTAIL_SYS_CONF_DIR + +- ilogtail_docker_file_path_config + +## 升级建议 + +1. **兼容模式**: 如需保持与 Logtail 的兼容性,请参考 [LoongCollector 的 Logtail 兼容模式使用指南](logtail-mode.md) + +2. **新版迁移**: 如果选择使用新版目录结构: + - 建议先备份原有配置和数据 + + - 按新版目录结构迁移文件 + + - 更新相关配置引用 + + - 验证服务正常运行 + +为确保平稳迁移，我们建议您： + +1. 先在测试环境进行充分验证 + +2. 选择业务低峰期进行升级 + +3. 做好配置和数据的备份 + +4. 逐步迁移，避免一次性升级所有实例 + +5. 密切监控日志采集状态 + +> **注意**: 迁移过程中请确保数据完整性,建议先在测试环境中进行测试，并非高峰期进行升级操作。 \ No newline at end of file diff --git a/pkg/config/global_config.go b/pkg/config/global_config.go index 487428fa8f..dbcee71a4b 100644 --- a/pkg/config/global_config.go +++ b/pkg/config/global_config.go @@ -38,6 +38,12 @@ type GlobalConfig struct { LoongcollectorDebugDir string // Directory to store loongcollector third party data. LoongcollectorThirdPartyDir string + // Log name of loongcollector plugin. + LoongcollectorPluginLogName string + // Tag of loongcollector version. + LoongcollectorVersionTag string + // Checkpoint file name of loongcollector plugin. + LoongcollectorCheckPointFile string // Network identification from loongcollector. HostIP string Hostname string @@ -59,18 +65,21 @@ var UserAgent = fmt.Sprintf("ilogtail/%v (%v)", BaseVersion, runtime.GOOS) // se func newGlobalConfig() (cfg GlobalConfig) { cfg = GlobalConfig{ - InputMaxFirstCollectDelayMs: 10000, // 10s - InputIntervalMs: 1000, // 1s - AggregatIntervalMs: 3000, - FlushIntervalMs: 3000, - DefaultLogQueueSize: 1000, - DefaultLogGroupQueueSize: 4, - LoongcollectorConfDir: "./conf/", - LoongcollectorLogDir: "./log/", - LoongcollectorDataDir: "./data/", - LoongcollectorDebugDir: "./debug/", - LoongcollectorThirdPartyDir: "./thirdparty/", - DelayStopSec: 300, + InputMaxFirstCollectDelayMs: 10000, // 10s + InputIntervalMs: 1000, // 1s + AggregatIntervalMs: 3000, + FlushIntervalMs: 3000, + DefaultLogQueueSize: 1000, + DefaultLogGroupQueueSize: 4, + LoongcollectorConfDir: "./conf/", + LoongcollectorLogDir: "./log/", + LoongcollectorPluginLogName: "go_plugin.LOG", + LoongcollectorVersionTag: "loongcollector_version", + LoongcollectorCheckPointFile: "go_plugin_checkpoint", + LoongcollectorDataDir: "./data/", + LoongcollectorDebugDir: "./debug/", + LoongcollectorThirdPartyDir: "./thirdparty/", + DelayStopSec: 300, } return } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 802502f41b..6b5e6f6e61 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -42,7 +42,7 @@ const ( asyncPattern = ` - + %s %s @@ -54,7 +54,7 @@ const ( syncPattern = ` - + %s %s @@ -325,7 +325,7 @@ func generateDefaultConfig() string { if memoryReceiverFlag { memoryReceiverFlagStr = "" } - return fmt.Sprintf(template, levelFlag, config.LoongcollectorGlobalConfig.LoongcollectorLogDir, consoleStr, memoryReceiverFlagStr) + return fmt.Sprintf(template, levelFlag, config.LoongcollectorGlobalConfig.LoongcollectorLogDir, config.LoongcollectorGlobalConfig.LoongcollectorPluginLogName, consoleStr, memoryReceiverFlagStr) } // Close the logger and recover the stdout and stderr diff --git a/pkg/logger/logger_test.go b/pkg/logger/logger_test.go index f3bcc9a0d9..7fc6cb0bd2 100644 --- a/pkg/logger/logger_test.go +++ b/pkg/logger/logger_test.go @@ -49,11 +49,11 @@ func init() { func clean() { _ = os.Remove(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorConfDir, "plugin_logger.xml")) - _ = os.Remove(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, "go_plugin.LOG")) + _ = os.Remove(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, config.LoongcollectorGlobalConfig.LoongcollectorPluginLogName)) } func readLog(index int) string { - bytes, _ := os.ReadFile(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, "go_plugin.LOG")) + bytes, _ := os.ReadFile(path.Join(config.LoongcollectorGlobalConfig.LoongcollectorLogDir, config.LoongcollectorGlobalConfig.LoongcollectorPluginLogName)) logs := strings.Split(string(bytes), "\n") if index > len(logs)-1 { return "" diff --git a/pkg/logtail/libGoPluginAdapter.so b/pkg/logtail/libGoPluginAdapter.so index 5c90a1d9aae716389fe8a173cd9be6067fe2e087..7c2d78f120cb22e3707d42f4e6cdec7089dc0c61 100755 GIT binary patch delta 45 zcmZ25lWD Date: Wed, 20 Nov 2024 16:55:31 +0800 Subject: [PATCH 05/11] chore: remove lc_scraper for auto metrics (#1897) --- core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp | 1 - core/prometheus/Constants.h | 1 - 2 files changed, 2 deletions(-) diff --git a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp index c5deeb149c..f6cbb0b79b 100644 --- a/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp +++ b/core/plugin/processor/inner/ProcessorPromRelabelMetricNative.cpp @@ -206,7 +206,6 @@ void ProcessorPromRelabelMetricNative::AddMetric(PipelineEventGroup& metricGroup metricEvent->SetValue(value); metricEvent->SetTimestamp(timestamp, nanoSec); metricEvent->SetTag(prometheus::NAME, name); - metricEvent->SetTag(prometheus::LC_SCRAPER, mLoongCollectorScraper); for (const auto& [k, v] : targetTags) { metricEvent->SetTag(k, v); } diff --git a/core/prometheus/Constants.h b/core/prometheus/Constants.h index f4a823b4f4..55c6712e12 100644 --- a/core/prometheus/Constants.h +++ b/core/prometheus/Constants.h @@ -98,7 +98,6 @@ const char* const SCRAPE_SAMPLES_POST_METRIC_RELABELING = "scrape_samples_post_m const char* const SCRAPE_SAMPLES_SCRAPED = "scrape_samples_scraped"; const char* const SCRAPE_TIMEOUT_SECONDS = "scrape_timeout_seconds"; const char* const UP = "up"; -const char* const LC_SCRAPER = "lc_scraper"; const char* const SCRAPE_TIMESTAMP_MILLISEC = "scrape_timestamp_millisec"; From 427b11bbb6c9ac6d9bb153804b6b101995a26e1d Mon Sep 17 00:00:00 2001 From: Kai <33246768+KayzzzZ@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:42:46 +0800 Subject: [PATCH 06/11] feat: support span serializer (#1903) * feat: support span serializer Signed-off-by: qianlu.kk --- core/constants/SpanConstants.cpp | 43 ++++++ core/constants/SpanConstants.h | 45 ++++++ core/constants/TagConstants.cpp | 3 + core/constants/TagConstants.h | 4 + core/models/SpanEvent.cpp | 90 ++++++++---- core/models/SpanEvent.h | 8 +- core/pipeline/serializer/SLSSerializer.cpp | 107 +++++++++++++- core/unittest/models/SpanEventUnittest.cpp | 20 +-- .../serializer/SLSSerializerUnittest.cpp | 134 ++++++++++++++++++ 9 files changed, 412 insertions(+), 42 deletions(-) create mode 100644 core/constants/SpanConstants.cpp create mode 100644 core/constants/SpanConstants.h diff --git a/core/constants/SpanConstants.cpp b/core/constants/SpanConstants.cpp new file mode 100644 index 0000000000..262c4f15b2 --- /dev/null +++ b/core/constants/SpanConstants.cpp @@ -0,0 +1,43 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "SpanConstants.h" + +namespace logtail { + + const std::string DEFAULT_TRACE_TAG_TRACE_ID = "traceId"; + const std::string DEFAULT_TRACE_TAG_SPAN_ID = "spanId"; + const std::string DEFAULT_TRACE_TAG_PARENT_ID = "parentSpanId"; + const std::string DEFAULT_TRACE_TAG_SPAN_NAME = "spanName"; + const std::string DEFAULT_TRACE_TAG_SERVICE_NAME = "serviceName"; + const std::string DEFAULT_TRACE_TAG_START_TIME_NANO = "startTime"; + const std::string DEFAULT_TRACE_TAG_END_TIME_NANO = "endTime"; + const std::string DEFAULT_TRACE_TAG_DURATION = "duration"; + const std::string DEFAULT_TRACE_TAG_ATTRIBUTES = "attributes"; + const std::string DEFAULT_TRACE_TAG_RESOURCE = "resources"; + const std::string DEFAULT_TRACE_TAG_LINKS = "links"; + const std::string DEFAULT_TRACE_TAG_EVENTS = "events"; + const std::string DEFAULT_TRACE_TAG_TIMESTAMP = "timestamp"; + const std::string DEFAULT_TRACE_TAG_STATUS_CODE = "statusCode"; + const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE = "statusMessage"; + const std::string DEFAULT_TRACE_TAG_SPAN_KIND = "kind"; + const std::string DEFAULT_TRACE_TAG_TRACE_STATE = "traceState"; + const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME = "name"; +#ifdef __ENTERPRISE__ + // for arms + const std::string DEFAULT_TRACE_TAG_APP_ID = "pid"; + const std::string DEFAULT_TRACE_TAG_IP = "ip"; +#endif + +} // namespace logtail \ No newline at end of file diff --git a/core/constants/SpanConstants.h b/core/constants/SpanConstants.h new file mode 100644 index 0000000000..36a9d2f030 --- /dev/null +++ b/core/constants/SpanConstants.h @@ -0,0 +1,45 @@ +/* + * Copyright 2024 iLogtail Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +namespace logtail { + extern const std::string DEFAULT_TRACE_TAG_TRACE_ID; + extern const std::string DEFAULT_TRACE_TAG_SPAN_ID; + extern const std::string DEFAULT_TRACE_TAG_PARENT_ID; + extern const std::string DEFAULT_TRACE_TAG_SPAN_NAME; + extern const std::string DEFAULT_TRACE_TAG_SERVICE_NAME; + extern const std::string DEFAULT_TRACE_TAG_START_TIME_NANO; + extern const std::string DEFAULT_TRACE_TAG_END_TIME_NANO; + extern const std::string DEFAULT_TRACE_TAG_DURATION; + extern const std::string DEFAULT_TRACE_TAG_ATTRIBUTES; + extern const std::string DEFAULT_TRACE_TAG_RESOURCE; + extern const std::string DEFAULT_TRACE_TAG_LINKS; + extern const std::string DEFAULT_TRACE_TAG_EVENTS; + extern const std::string DEFAULT_TRACE_TAG_TIMESTAMP; + extern const std::string DEFAULT_TRACE_TAG_STATUS_CODE; + extern const std::string DEFAULT_TRACE_TAG_STATUS_MESSAGE; + extern const std::string DEFAULT_TRACE_TAG_SPAN_KIND; + extern const std::string DEFAULT_TRACE_TAG_TRACE_STATE; + extern const std::string DEFAULT_TRACE_TAG_SPAN_EVENT_NAME ; +#ifdef __ENTERPRISE__ + // for arms + extern const std::string DEFAULT_TRACE_TAG_APP_ID; + extern const std::string DEFAULT_TRACE_TAG_IP; +#endif + +} // namespace logtail \ No newline at end of file diff --git a/core/constants/TagConstants.cpp b/core/constants/TagConstants.cpp index edd7ea4e55..4d4a865192 100644 --- a/core/constants/TagConstants.cpp +++ b/core/constants/TagConstants.cpp @@ -63,4 +63,7 @@ namespace logtail { const std::string DEFAULT_METRIC_TAG_CONTAINER_IP = DEFAULT_TAG_CONTAINER_IP; const std::string DEFAULT_METRIC_TAG_IMAGE_NAME = DEFAULT_TAG_IMAGE_NAME; +////////////////////////// TRACE //////////////////////// + + } // namespace logtail \ No newline at end of file diff --git a/core/constants/TagConstants.h b/core/constants/TagConstants.h index 52d4213961..0b83d4092d 100644 --- a/core/constants/TagConstants.h +++ b/core/constants/TagConstants.h @@ -44,4 +44,8 @@ namespace logtail { extern const std::string DEFAULT_METRIC_TAG_CONTAINER_IP; extern const std::string DEFAULT_METRIC_TAG_IMAGE_NAME; +////////////////////////// TRACE //////////////////////// + + + } // namespace logtail \ No newline at end of file diff --git a/core/models/SpanEvent.cpp b/core/models/SpanEvent.cpp index 812ccbe54b..4396a92860 100644 --- a/core/models/SpanEvent.cpp +++ b/core/models/SpanEvent.cpp @@ -15,6 +15,7 @@ */ #include "models/SpanEvent.h" +#include "constants/SpanConstants.h" using namespace std; @@ -75,16 +76,15 @@ size_t SpanEvent::SpanLink::DataSize() const { return mTraceId.size() + mSpanId.size() + mTraceState.size() + mTags.DataSize(); } -#ifdef APSARA_UNIT_TEST_MAIN Json::Value SpanEvent::SpanLink::ToJson() const { Json::Value root; - root["traceId"] = mTraceId.to_string(); - root["spanId"] = mSpanId.to_string(); + root[DEFAULT_TRACE_TAG_TRACE_ID] = mTraceId.to_string(); + root[DEFAULT_TRACE_TAG_SPAN_ID] = mSpanId.to_string(); if (!mTraceState.empty()) { - root["traceState"] = mTraceState.to_string(); + root[DEFAULT_TRACE_TAG_TRACE_STATE] = mTraceState.to_string(); } if (!mTags.mInner.empty()) { - Json::Value& tags = root["tags"]; + Json::Value& tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& tag : mTags.mInner) { tags[tag.first.to_string()] = tag.second.to_string(); } @@ -92,17 +92,18 @@ Json::Value SpanEvent::SpanLink::ToJson() const { return root; } +#ifdef APSARA_UNIT_TEST_MAIN void SpanEvent::SpanLink::FromJson(const Json::Value& value) { - SetTraceId(value["traceId"].asString()); - SetSpanId(value["spanId"].asString()); - if (value.isMember("traceState")) { - string s = value["traceState"].asString(); + SetTraceId(value[DEFAULT_TRACE_TAG_TRACE_ID].asString()); + SetSpanId(value[DEFAULT_TRACE_TAG_SPAN_ID].asString()); + if (value.isMember(DEFAULT_TRACE_TAG_TRACE_STATE)) { + string s = value[DEFAULT_TRACE_TAG_TRACE_STATE].asString(); if (!s.empty()) { SetTraceState(s); } } - if (value.isMember("tags")) { - Json::Value tags = value["tags"]; + if (value.isMember(DEFAULT_TRACE_TAG_ATTRIBUTES)) { + Json::Value tags = value[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& key : tags.getMemberNames()) { SetTag(key, tags[key].asString()); } @@ -155,13 +156,12 @@ size_t SpanEvent::InnerEvent::DataSize() const { return sizeof(decltype(mTimestampNs)) + mName.size() + mTags.DataSize(); } -#ifdef APSARA_UNIT_TEST_MAIN Json::Value SpanEvent::InnerEvent::ToJson() const { Json::Value root; - root["name"] = mName.to_string(); - root["timestampNs"] = static_cast(mTimestampNs); + root[DEFAULT_TRACE_TAG_SPAN_EVENT_NAME] = mName.to_string(); + root[DEFAULT_TRACE_TAG_TIMESTAMP] = static_cast(mTimestampNs); if (!mTags.mInner.empty()) { - Json::Value& tags = root["tags"]; + Json::Value& tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& tag : mTags.mInner) { tags[tag.first.to_string()] = tag.second.to_string(); } @@ -169,11 +169,12 @@ Json::Value SpanEvent::InnerEvent::ToJson() const { return root; } +#ifdef APSARA_UNIT_TEST_MAIN void SpanEvent::InnerEvent::FromJson(const Json::Value& value) { - SetName(value["name"].asString()); - SetTimestampNs(value["timestampNs"].asUInt64()); - if (value.isMember("tags")) { - Json::Value tags = value["tags"]; + SetName(value[DEFAULT_TRACE_TAG_SPAN_EVENT_NAME].asString()); + SetTimestampNs(value[DEFAULT_TRACE_TAG_TIMESTAMP].asUInt64()); + if (value.isMember(DEFAULT_TRACE_TAG_ATTRIBUTES)) { + Json::Value tags = value[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& key : tags.getMemberNames()) { SetTag(key, tags[key].asString()); } @@ -350,19 +351,19 @@ Json::Value SpanEvent::ToJson(bool enableEventMeta) const { root["startTimeNs"] = static_cast(mStartTimeNs); root["endTimeNs"] = static_cast(mEndTimeNs); if (!mTags.mInner.empty()) { - Json::Value& tags = root["tags"]; + Json::Value& tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& tag : mTags.mInner) { tags[tag.first.to_string()] = tag.second.to_string(); } } if (!mEvents.empty()) { - Json::Value& events = root["events"]; + Json::Value& events = root[DEFAULT_TRACE_TAG_EVENTS]; for (const auto& event : mEvents) { events.append(event.ToJson()); } } if (!mLinks.empty()) { - Json::Value& links = root["links"]; + Json::Value& links = root[DEFAULT_TRACE_TAG_LINKS]; for (const auto& link : mLinks) { links.append(link.ToJson()); } @@ -405,21 +406,21 @@ bool SpanEvent::FromJson(const Json::Value& root) { } SetStartTimeNs(root["startTimeNs"].asUInt64()); SetEndTimeNs(root["endTimeNs"].asUInt64()); - if (root.isMember("tags")) { - Json::Value tags = root["tags"]; + if (root.isMember(DEFAULT_TRACE_TAG_ATTRIBUTES)) { + Json::Value tags = root[DEFAULT_TRACE_TAG_ATTRIBUTES]; for (const auto& key : tags.getMemberNames()) { SetTag(key, tags[key].asString()); } } - if (root.isMember("events")) { - Json::Value events = root["events"]; + if (root.isMember(DEFAULT_TRACE_TAG_EVENTS)) { + Json::Value events = root[DEFAULT_TRACE_TAG_EVENTS]; for (const auto& event : events) { InnerEvent* e = AddEvent(); e->FromJson(event); } } - if (root.isMember("links")) { - Json::Value links = root["links"]; + if (root.isMember(DEFAULT_TRACE_TAG_LINKS)) { + Json::Value links = root[DEFAULT_TRACE_TAG_LINKS]; for (const auto& link : links) { SpanLink* l = AddLink(); l->FromJson(link); @@ -438,4 +439,37 @@ bool SpanEvent::FromJson(const Json::Value& root) { } #endif +const static std::string sSpanStatusCodeUnSet = "UNSET"; +const static std::string sSpanStatusCodeOk = "OK"; +const static std::string sSpanStatusCodeError = "ERROR"; + +const std::string& GetStatusString(SpanEvent::StatusCode status) { + switch (status) { + case SpanEvent::StatusCode::Unset: return sSpanStatusCodeUnSet; + case SpanEvent::StatusCode::Ok: return sSpanStatusCodeOk; + case SpanEvent::StatusCode::Error: return sSpanStatusCodeError; + default: return sSpanStatusCodeUnSet; + } +} + +const static std::string sSpanKindUnspecified = "unspecified"; +const static std::string sSpanKindInternal = "internal"; +const static std::string sSpanKindServer = "server"; +const static std::string sSpanKindClient = "client"; +const static std::string sSpanKindProducer = "producer"; +const static std::string sSpanKindConsumer = "consumer"; +const static std::string sSpanKindUnknown = "unknown"; + +const std::string& GetKindString(SpanEvent::Kind kind) { + switch (kind) { + case SpanEvent::Kind::Unspecified: return sSpanKindUnspecified; + case SpanEvent::Kind::Internal: return sSpanKindInternal; + case SpanEvent::Kind::Server: return sSpanKindServer; + case SpanEvent::Kind::Client: return sSpanKindClient; + case SpanEvent::Kind::Producer: return sSpanKindProducer; + case SpanEvent::Kind::Consumer: return sSpanKindConsumer; + default: return sSpanKindUnknown; + } +} + } // namespace logtail diff --git a/core/models/SpanEvent.h b/core/models/SpanEvent.h index 584d64a5d6..780664eb1a 100644 --- a/core/models/SpanEvent.h +++ b/core/models/SpanEvent.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "common/memory/SourceBuffer.h" #include "models/PipelineEvent.h" @@ -67,8 +68,8 @@ class SpanEvent : public PipelineEvent { size_t DataSize() const; -#ifdef APSARA_UNIT_TEST_MAIN Json::Value ToJson() const; +#ifdef APSARA_UNIT_TEST_MAIN void FromJson(const Json::Value& value); #endif @@ -107,8 +108,8 @@ class SpanEvent : public PipelineEvent { size_t DataSize() const; -#ifdef APSARA_UNIT_TEST_MAIN Json::Value ToJson() const; +#ifdef APSARA_UNIT_TEST_MAIN void FromJson(const Json::Value& value); #endif @@ -214,4 +215,7 @@ class SpanEvent : public PipelineEvent { #endif }; +const std::string& GetStatusString(SpanEvent::StatusCode status); +const std::string& GetKindString(SpanEvent::Kind kind); + } // namespace logtail diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index 9ffb6b1541..6b9ec3888d 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -15,9 +15,12 @@ #include "pipeline/serializer/SLSSerializer.h" #include "common/Flags.h" +#include "constants/SpanConstants.h" #include "common/compression/CompressType.h" #include "plugin/flusher/sls/FlusherSLS.h" #include "protobuf/sls/LogGroupSerializer.h" +#include +#include DECLARE_FLAG_INT32(max_send_log_group_size); @@ -25,6 +28,29 @@ using namespace std; namespace logtail { +std::string SerializeSpanLinksToString(const SpanEvent& event) { + if (event.GetLinks().empty()) { + return ""; + } + Json::Value jsonLinks(Json::arrayValue); + for (const auto& link : event.GetLinks()) { + jsonLinks.append(link.ToJson()); + } + Json::StreamWriterBuilder writer; + return Json::writeString(writer, jsonLinks); +} +std::string SerializeSpanEventsToString(const SpanEvent& event) { + if (event.GetEvents().empty()) { + return ""; + } + Json::Value jsonEvents(Json::arrayValue); + for (const auto& event : event.GetEvents()) { + jsonEvents.append(event.ToJson()); + } + Json::StreamWriterBuilder writer; + return Json::writeString(writer, jsonEvents); +} + template <> bool Serializer>::DoSerialize(vector&& p, std::string& output, @@ -68,9 +94,10 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri // caculate serialized logGroup size first, where some critical results can be cached vector logSZ(group.mEvents.size()); vector> metricEventContentCache(group.mEvents.size()); + vector> spanEventContentCache(group.mEvents.size()); size_t logGroupSZ = 0; switch (eventType) { - case PipelineEvent::Type::LOG: + case PipelineEvent::Type::LOG:{ for (size_t i = 0; i < group.mEvents.size(); ++i) { const auto& e = group.mEvents[i].Cast(); if (e.Empty()) { @@ -83,7 +110,8 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logGroupSZ += GetLogSize(contentSZ, enableNs && e.GetTimestampNanosecond(), logSZ[i]); } break; - case PipelineEvent::Type::METRIC: + } + case PipelineEvent::Type::METRIC:{ for (size_t i = 0; i < group.mEvents.size(); ++i) { const auto& e = group.mEvents[i].Cast(); if (e.Is()) { @@ -107,7 +135,51 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri logGroupSZ += GetLogSize(contentSZ, false, logSZ[i]); } break; + } case PipelineEvent::Type::SPAN: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& e = group.mEvents[i].Cast(); + size_t contentSZ = 0; + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_TRACE_ID.size(), e.GetTraceId().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_SPAN_ID.size(), e.GetSpanId().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_PARENT_ID.size(), e.GetParentSpanId().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_SPAN_NAME.size(), e.GetName().size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_SPAN_KIND.size(), GetKindString(e.GetKind()).size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_STATUS_CODE.size(), GetStatusString(e.GetStatus()).size()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_TRACE_STATE.size(), e.GetTraceState().size()); + // + // set tags and scope tags + Json::Value jsonVal; + for (auto it = e.TagsBegin(); it != e.TagsEnd(); ++it) { + jsonVal[it->first.to_string()] = it->second.to_string(); + } + for (auto it = e.ScopeTagsBegin(); it != e.ScopeTagsEnd(); ++it) { + jsonVal[it->first.to_string()] = it->second.to_string(); + } + Json::StreamWriterBuilder writer; + std::string attrString = Json::writeString(writer, jsonVal); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_ATTRIBUTES.size(), attrString.size()); + spanEventContentCache[i][0] = std::move(attrString); + + auto linkString = SerializeSpanLinksToString(e); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_LINKS.size(), linkString.size()); + spanEventContentCache[i][1] = std::move(linkString); + auto eventString = SerializeSpanEventsToString(e); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_EVENTS.size(), eventString.size()); + spanEventContentCache[i][2] = std::move(eventString); + + // time related + auto startTsNs = std::to_string(e.GetStartTimeNs()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_START_TIME_NANO.size(), startTsNs.size()); + spanEventContentCache[i][3] = std::move(startTsNs); + auto endTsNs = std::to_string(e.GetEndTimeNs()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_END_TIME_NANO.size(), endTsNs.size()); + spanEventContentCache[i][4] = std::move(endTsNs); + auto durationNs = std::to_string(e.GetEndTimeNs() - e.GetStartTimeNs()); + contentSZ += GetLogContentSize(DEFAULT_TRACE_TAG_DURATION.size(), durationNs.size()); + spanEventContentCache[i][5] = std::move(durationNs); + logGroupSZ += GetLogSize(contentSZ, false, logSZ[i]); + } break; case PipelineEvent::Type::RAW: for (size_t i = 0; i < group.mEvents.size(); ++i) { @@ -174,6 +246,37 @@ bool SLSEventGroupSerializer::Serialize(BatchedEvents&& group, string& res, stri } break; case PipelineEvent::Type::SPAN: + for (size_t i = 0; i < group.mEvents.size(); ++i) { + const auto& spanEvent = group.mEvents[i].Cast(); + + serializer.StartToAddLog(logSZ[i]); + serializer.AddLogTime(spanEvent.GetTimestamp()); + // set trace_id span_id span_kind status etc + serializer.AddLogContent(DEFAULT_TRACE_TAG_TRACE_ID, spanEvent.GetTraceId()); + serializer.AddLogContent(DEFAULT_TRACE_TAG_SPAN_ID, spanEvent.GetSpanId()); + serializer.AddLogContent(DEFAULT_TRACE_TAG_PARENT_ID, spanEvent.GetParentSpanId()); + // span_name + serializer.AddLogContent(DEFAULT_TRACE_TAG_SPAN_NAME, spanEvent.GetName()); + // span_kind + serializer.AddLogContent(DEFAULT_TRACE_TAG_SPAN_KIND, GetKindString(spanEvent.GetKind())); + // status_code + serializer.AddLogContent(DEFAULT_TRACE_TAG_STATUS_CODE, GetStatusString(spanEvent.GetStatus())); + // trace state + serializer.AddLogContent(DEFAULT_TRACE_TAG_TRACE_STATE, spanEvent.GetTraceState()); + + serializer.AddLogContent(DEFAULT_TRACE_TAG_ATTRIBUTES, spanEventContentCache[i][0]); + + serializer.AddLogContent(DEFAULT_TRACE_TAG_LINKS, spanEventContentCache[i][1]); + serializer.AddLogContent(DEFAULT_TRACE_TAG_EVENTS, spanEventContentCache[i][2]); + + // start_time + serializer.AddLogContent(DEFAULT_TRACE_TAG_START_TIME_NANO, spanEventContentCache[i][3]); + // end_time + serializer.AddLogContent(DEFAULT_TRACE_TAG_END_TIME_NANO, spanEventContentCache[i][4]); + // duration + serializer.AddLogContent(DEFAULT_TRACE_TAG_DURATION, spanEventContentCache[i][5]); + + } break; case PipelineEvent::Type::RAW: for (size_t i = 0; i < group.mEvents.size(); ++i) { diff --git a/core/unittest/models/SpanEventUnittest.cpp b/core/unittest/models/SpanEventUnittest.cpp index 768097a65b..693538324d 100644 --- a/core/unittest/models/SpanEventUnittest.cpp +++ b/core/unittest/models/SpanEventUnittest.cpp @@ -319,13 +319,13 @@ void SpanEventUnittest::TestToJson() { "kind": 3, "startTimeNs": 1715826723000000000, "endTimeNs": 1715826725000000000, - "tags": { + "attributes": { "key1": "value1" }, "events": [ { "name": "test_event", - "timestampNs": 1715826724000000000 + "timestamp": 1715826724000000000 } ], "links": [ @@ -358,13 +358,13 @@ void SpanEventUnittest::TestFromJson() { "kind": 3, "startTimeNs": 1715826723000000000, "endTimeNs": 1715826725000000000, - "tags": { + "attributes": { "key1": "value1" }, "events": [ { "name": "test_event", - "timestampNs": 1715826724000000000 + "timestamp": 1715826724000000000 } ], "links": [ @@ -513,8 +513,8 @@ void InnerEventUnittest::TestToJson() { Json::Value eventJson; string eventStr = R"({ "name": "test", - "timestampNs": 1715826723000000000, - "tags": { + "timestamp": 1715826723000000000, + "attributes": { "key1": "value1" } })"; @@ -528,8 +528,8 @@ void InnerEventUnittest::TestFromJson() { Json::Value eventJson; string eventStr = R"({ "name": "test", - "timestampNs": 1715826723000000000, - "tags": { + "timestamp": 1715826723000000000, + "attributes": { "key1": "value1" } })"; @@ -659,7 +659,7 @@ void SpanLinkUnittest::TestToJson() { "traceId": "test_trace_id", "spanId": "test_span_id", "traceState": "normal", - "tags": { + "attributes": { "key1": "value1" } })"; @@ -675,7 +675,7 @@ void SpanLinkUnittest::TestFromJson() { "traceId": "test_trace_id", "spanId": "test_span_id", "traceState": "normal", - "tags": { + "attributes": { "key1": "value1" } })"; diff --git a/core/unittest/serializer/SLSSerializerUnittest.cpp b/core/unittest/serializer/SLSSerializerUnittest.cpp index 603f95f953..75a36a307d 100644 --- a/core/unittest/serializer/SLSSerializerUnittest.cpp +++ b/core/unittest/serializer/SLSSerializerUnittest.cpp @@ -41,6 +41,7 @@ class SLSSerializerUnittest : public ::testing::Test { BatchedEvents CreateBatchedMetricEvents(bool enableNanosecond, uint32_t nanoTimestamp, bool emptyValue, bool onlyOneTag); BatchedEvents CreateBatchedRawEvents(bool enableNanosecond, bool emptyContent); + BatchedEvents CreateBatchedSpanEvents(); static unique_ptr sFlusher; @@ -215,6 +216,86 @@ void SLSSerializerUnittest::TestSerializeEventGroup() { } { // span + string res, errorMsg; + auto events = CreateBatchedSpanEvents(); + APSARA_TEST_EQUAL(events.mEvents.size(), 1); + APSARA_TEST_TRUE(events.mEvents[0]->GetType() == PipelineEvent::Type::SPAN); + APSARA_TEST_TRUE(serializer.DoSerialize(std::move(events), res, errorMsg)); + sls_logs::LogGroup logGroup; + APSARA_TEST_TRUE(logGroup.ParseFromString(res)); + APSARA_TEST_EQUAL(1, logGroup.logs_size()); + APSARA_TEST_EQUAL(13, logGroup.logs(0).contents_size()); + // traceid + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).key(), "traceId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(0).value(), "trace-1-2-3-4-5"); + // span id + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).key(), "spanId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(1).value(), "span-1-2-3-4-5"); + // parent span id + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).key(), "parentSpanId"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(2).value(), "parent-1-2-3-4-5"); + // spanName + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).key(), "spanName"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(3).value(), "/oneagent/qianlu/local/1"); + // kind + APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).key(), "kind"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(4).value(), "client"); + // code + APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).key(), "statusCode"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(5).value(), "OK"); + // traceState + APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).key(), "traceState"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(6).value(), "test-state"); + // attributes + APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).key(), "attributes"); + auto attrs = logGroup.logs(0).contents(7).value(); + Json::Value jsonVal; + Json::CharReaderBuilder readerBuilder; + std::string errs; + + std::istringstream s(attrs); + bool ret = Json::parseFromStream(readerBuilder, s, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 10); + APSARA_TEST_EQUAL(jsonVal["rpcType"].asString(), "25"); + APSARA_TEST_EQUAL(jsonVal["scope-tag-0"].asString(), "scope-value-0"); + // APSARA_TEST_EQUAL(logGroup.logs(0).contents(7).value(), ""); + // links + APSARA_TEST_EQUAL(logGroup.logs(0).contents(8).key(), "links"); + + auto linksStr = logGroup.logs(0).contents(8).value(); + + std::istringstream ss(linksStr); + ret = Json::parseFromStream(readerBuilder, ss, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 1); + for (auto& link : jsonVal) { + APSARA_TEST_EQUAL(link["spanId"].asString(), "inner-link-spanid"); + APSARA_TEST_EQUAL(link["traceId"].asString(), "inner-link-traceid"); + APSARA_TEST_EQUAL(link["traceState"].asString(), "inner-link-trace-state"); + } + // events + APSARA_TEST_EQUAL(logGroup.logs(0).contents(9).key(), "events"); + auto eventsStr = logGroup.logs(0).contents(9).value(); + std::istringstream sss(eventsStr); + ret = Json::parseFromStream(readerBuilder, sss, &jsonVal, &errs); + APSARA_TEST_TRUE(ret); + APSARA_TEST_EQUAL(jsonVal.size(), 1); + for (auto& event : jsonVal) { + APSARA_TEST_EQUAL(event["name"].asString(), "inner-event"); + APSARA_TEST_EQUAL(event["timestamp"].asString(), "1000"); + } + // start + APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).key(), "startTime"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(10).value(), "1000"); + + // end + APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).key(), "endTime"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(11).value(), "2000"); + + // duration + APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).key(), "duration"); + APSARA_TEST_EQUAL(logGroup.logs(0).contents(12).value(), "1000"); } { // raw @@ -391,6 +472,59 @@ BatchedEvents SLSSerializerUnittest::CreateBatchedRawEvents(bool enableNanosecon return batch; } +BatchedEvents SLSSerializerUnittest::CreateBatchedSpanEvents() { + PipelineEventGroup group(make_shared()); + group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); + group.SetTag(LOG_RESERVED_KEY_SOURCE, "source"); + group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "aaa"); + group.SetTag(LOG_RESERVED_KEY_PACKAGE_ID, "bbb"); + auto now = std::chrono::system_clock::now(); + auto duration = now.time_since_epoch(); + auto seconds = std::chrono::duration_cast(duration).count(); + // auto nano = std::chrono::duration_cast(duration).count(); + StringBuffer b = group.GetSourceBuffer()->CopyString(string("pack_id")); + group.SetMetadataNoCopy(EventGroupMetaKey::SOURCE_ID, StringView(b.data, b.size)); + group.SetExactlyOnceCheckpoint(RangeCheckpointPtr(new RangeCheckpoint)); + SpanEvent* spanEvent = group.AddSpanEvent(); + spanEvent->SetScopeTag(std::string("scope-tag-0"), std::string("scope-value-0")); + spanEvent->SetTag(std::string("workloadName"), std::string("arms-oneagent-test-ql")); + spanEvent->SetTag(std::string("workloadKind"), std::string("faceless")); + spanEvent->SetTag(std::string("source_ip"), std::string("10.54.0.33")); + spanEvent->SetTag(std::string("host"), std::string("10.54.0.33")); + spanEvent->SetTag(std::string("rpc"), std::string("/oneagent/qianlu/local/1")); + spanEvent->SetTag(std::string("rpcType"), std::string("25")); + spanEvent->SetTag(std::string("callType"), std::string("http-client")); + spanEvent->SetTag(std::string("statusCode"), std::string("200")); + spanEvent->SetTag(std::string("version"), std::string("HTTP1.1")); + auto innerEvent = spanEvent->AddEvent(); + innerEvent->SetTag(std::string("innner-event-key-0"), std::string("inner-event-value-0")); + innerEvent->SetTag(std::string("innner-event-key-1"), std::string("inner-event-value-1")); + innerEvent->SetName("inner-event"); + innerEvent->SetTimestampNs(1000); + auto innerLink = spanEvent->AddLink(); + innerLink->SetTag(std::string("innner-link-key-0"), std::string("inner-link-value-0")); + innerLink->SetTag(std::string("innner-link-key-1"), std::string("inner-link-value-1")); + innerLink->SetTraceId("inner-link-traceid"); + innerLink->SetSpanId("inner-link-spanid"); + innerLink->SetTraceState("inner-link-trace-state"); + spanEvent->SetName("/oneagent/qianlu/local/1"); + spanEvent->SetKind(SpanEvent::Kind::Client); + spanEvent->SetStatus(SpanEvent::StatusCode::Ok); + spanEvent->SetSpanId("span-1-2-3-4-5"); + spanEvent->SetTraceId("trace-1-2-3-4-5"); + spanEvent->SetParentSpanId("parent-1-2-3-4-5"); + spanEvent->SetTraceState("test-state"); + spanEvent->SetStartTimeNs(1000); + spanEvent->SetEndTimeNs(2000); + spanEvent->SetTimestamp(seconds); + BatchedEvents batch(std::move(group.MutableEvents()), + std::move(group.GetSizedTags()), + std::move(group.GetSourceBuffer()), + group.GetMetadata(EventGroupMetaKey::SOURCE_ID), + std::move(group.GetExactlyOnceCheckpoint())); + return batch; +} + UNIT_TEST_CASE(SLSSerializerUnittest, TestSerializeEventGroup) UNIT_TEST_CASE(SLSSerializerUnittest, TestSerializeEventGroupList) From 7c80c3b2851966c8caad0329b4966cdf058a0065 Mon Sep 17 00:00:00 2001 From: henryzhx8 Date: Fri, 22 Nov 2024 13:38:10 +0800 Subject: [PATCH 07/11] add development doc for native plugin (#1907) --- core/plugin/flusher/sls/SLSResponse.cpp | 2 +- docs/cn/SUMMARY.md | 10 +- .../how-to-write-native-flusher-plugins.md | 209 ++++++++++++++++++ .../how-to-write-native-input-plugins.md | 98 ++++++++ 4 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 docs/cn/developer-guide/plugin-development/how-to-write-native-flusher-plugins.md create mode 100644 docs/cn/developer-guide/plugin-development/how-to-write-native-input-plugins.md diff --git a/core/plugin/flusher/sls/SLSResponse.cpp b/core/plugin/flusher/sls/SLSResponse.cpp index b55896aa1a..d9fa405479 100644 --- a/core/plugin/flusher/sls/SLSResponse.cpp +++ b/core/plugin/flusher/sls/SLSResponse.cpp @@ -34,7 +34,7 @@ bool SLSResponse::Parse(const HttpResponse& response) { mStatusCode = response.GetStatusCode(); if (mStatusCode == 0) { - mErrorCode = sdk::LOG_REQUEST_TIMEOUT; + mErrorCode = sdk::LOGE_REQUEST_TIMEOUT; mErrorMsg = "Request timeout"; } else if (mStatusCode != 200) { try { diff --git a/docs/cn/SUMMARY.md b/docs/cn/SUMMARY.md index f830edbb95..9a99de227d 100644 --- a/docs/cn/SUMMARY.md +++ b/docs/cn/SUMMARY.md @@ -150,10 +150,12 @@ * [Checkpoint接口](developer-guide/plugin-development/checkpoint-api.md) * [Logger接口](developer-guide/plugin-development/logger-api.md) * [自监控指标接口](developer-guide/plugin-development/plugin-self-monitor-guide.md) - * [如何开发Input插件](developer-guide/plugin-development/how-to-write-input-plugins.md) - * [如何开发Processor插件](developer-guide/plugin-development/how-to-write-processor-plugins.md) - * [如何开发Aggregator插件](developer-guide/plugin-development/how-to-write-aggregator-plugins.md) - * [如何开发Flusher插件](developer-guide/plugin-development/how-to-write-flusher-plugins.md) + * [如何开发原生Input插件](developer-guide/plugin-development/how-to-write-native-input-plugins.md) + * [如何开发原生Flusher插件](developer-guide/plugin-development/how-to-write-native-flusher-plugins.md) + * [如何开发扩展Input插件](developer-guide/plugin-development/how-to-write-input-plugins.md) + * [如何开发扩展Processor插件](developer-guide/plugin-development/how-to-write-processor-plugins.md) + * [如何开发扩展Aggregator插件](developer-guide/plugin-development/how-to-write-aggregator-plugins.md) + * [如何开发扩展Flusher插件](developer-guide/plugin-development/how-to-write-flusher-plugins.md) * [如何生成插件文档](developer-guide/plugin-development/how-to-genernate-plugin-docs.md) * [插件文档规范](docs/cn/developer-guide/plugin-development/plugin-doc-templete.md) * [纯插件模式启动](developer-guide/plugin-development/pure-plugin-start.md) diff --git a/docs/cn/developer-guide/plugin-development/how-to-write-native-flusher-plugins.md b/docs/cn/developer-guide/plugin-development/how-to-write-native-flusher-plugins.md new file mode 100644 index 0000000000..031b1ed76c --- /dev/null +++ b/docs/cn/developer-guide/plugin-development/how-to-write-native-flusher-plugins.md @@ -0,0 +1,209 @@ +# 如何开发原生Flusher插件 + +## 接口定义 + +```c++ +class Flusher : public Plugin { +public: + // 用于初始化插件参数，同时根据参数初始化Flusher级的组件 + virtual bool Init(const Json::Value& config, Json::Value& optionalGoPipeline) = 0; + virtual bool Start(); + virtual bool Stop(bool isPipelineRemoving); + // 用于将处理插件的输出经过聚合、序列化和压缩处理后，放入发送队列 + virtual void Send(PipelineEventGroup&& g) = 0; + // 用于将聚合组件内指定聚合队列内的数据进行强制发送 + virtual void Flush(size_t key) = 0; + // 用于将聚合组件内的所有数据进行强制发送 + virtual void FlushAll() = 0; +}; +``` + +对于使用Http协议发送数据的Flusher，进一步定义了HttpFlusher，接口如下： + +```c++ +class HttpFlusher : public Flusher { +public: + // 用于将待发送数据打包成http请求 + virtual bool BuildRequest(SenderQueueItem* item, std::unique_ptr& req, bool* keepItem) const = 0; + // 用于发送完成后进行记录和处理 + virtual void OnSendDone(const HttpResponse& response, SenderQueueItem* item) = 0; +}; +``` + +## Flusher级组件 + +### 聚合（必选） + +* 作用：将多个小的PipelineEventGroup根据tag异同合并成一个大的group，提升发送效率 + +* 参数： + +可以在flusher的参数中配置Batch字段，该字段的类型为map，其中允许包含的字段如下： + +| **名称** | **类型** | **默认值** | **说明** | +| --- | --- | --- | --- | +| MinCnt | uint | 每个Flusher自定义 | 每个聚合队列最少包含的event数量 | +| MinSizeBytes | uint | 每个Flusher自定义 | 每个聚合队列最小的尺寸 | +| TimeoutSecs | uint | 每个Flusher自定义 | 每个聚合队列在第一个event加入后，在被输出前最多等待的时间 | + +* 类接口： + +```c++ +template +class Batcher { +public: + bool Init(const Json::Value& config, + Flusher* flusher, + const DefaultFlushStrategyOptions& strategy, + bool enableGroupBatch = false); + void Add(PipelineEventGroup&& g, std::vector& res); + void FlushQueue(size_t key, BatchedEventsList& res); + void FlushAll(std::vector& res); +} +``` + +### 序列化（必选） + +* 作用：对聚合模块的输出进行序列化，分为2个层级： + + * event级：对每一个event单独进行序列化 + + * event group级：对多个event进行批量的序列化 + +* 类接口： + +```c++ +// T: PipelineEventPtr, BatchedEvents, BatchedEventsList +template +class Serializer { +private: + virtual bool Serialize(T&& p, std::string& res, std::string& errorMsg) = 0; +}; + +using EventSerializer = Serializer; +using EventGroupSerializer = Serializer; +``` + +### 压缩（可选） + +* 作用：对序列化后的结果进行压缩 + +* 类接口： + +```c++ +class Compressor { +private: + virtual bool Compress(const std::string& input, std::string& output, std::string& errorMsg) = 0; +}; +``` + +## 开发步骤 + +下面以开发一个HttpFlusher为例，说明整个开发步骤： + +1. 在plugin/flusher目录下新建一个Flusherxxx.h和Flusherxxx.cpp文件，用于派生HttpFlusher接口生成具体的插件类； + +2. 在Flusherxxx.h文件中定义新的输出插件类Flusherxxx，满足以下要求： + + a. 所有的可配置参数的权限为public，其余参数的权限均为private + + b. 新增一个聚合组件：`Batcher<> mBatcher;` + + c. 新增一个序列化组件：`std::unique_ptr mSerializer;`，其中T为`EventSerializer`和`EventGroupSerializer`中的一种 + + d. 如果需要压缩，则新增一个压缩组件：`std::unique_ptr mCompressor;` + +3. 在pipeline/serializer目录下新建一个xxxSerializer.h和xxxSerializer.cpp文件，用于派生`Serializer` 接口生成具体类； + +4. （可选）如果需要压缩组件，且现有压缩组件库中没有所需算法，则新增一个压缩组件： + + a. 在common/compression/CompressType.h文件中，扩展CompressType类用以标识新的压缩算法； + + b. 在common/compression目录下新建一个xxxCompressor.h和xxxCompressor.cpp文件，用于派生`Compressor`接口生成具体类； + + c. 在common/compression/CompressorFactory.cpp文件的各个函数中注册该压缩组件； + +5. 在Flusherxxx.cpp文件中实现插件类 + + a. `Init`函数： + + i. 根据入参初始化插件，针对非法参数，根据非法程度和影响决定是跳过该参数、使用默认值或直接拒绝加载插件。 + + ii. 调用相关函数完成聚合、序列化和压缩组件的初始化 + + b. `SerializeAndPush（BatchedEventsList&&）`函数： + + ```c++ + void Flusherxxx::SerializeAndPush(BatchedEventsList&& groupLists) { + // 1. 使用mSerializer->Serialize函数对入参序列化 + // 2. 如果需要压缩，则使用mCompressor->Compress函数对序列化结果进行压缩 + // 3. 构建发送队列元素，其中， + // a. data为待发送内容 + // b. 如果没用压缩组件，则rawSize=data.size();否则，rawSize为压缩前（序列化后）数据的长度 + // c. mLogstoreKey为发送队列的key + auto item = make_unique(std::move(data), + rawSize, + this, + mQueueKey); + Flusher::PushToQueue(std::move(item)); + } + ``` + + c. `SerializeAndPush（vector&&）`函数： + + ```c++ + void Flusherxxx::SerializeAndPush(vector&& groupLists) { + for (auto& groupList : groupLists) { + SerializeAndPush(std::move(groupList)); + } + } + ``` + + d. `Send`函数： + + ```c++ + void Flusherxxx::Send(PipelineEventGroup&& g) { + vector res; + mBatcher.Add(std::move(g), res); + SerializeAndPush(std::move(res)); + } + ``` + + e. `Flush`函数： + + ```c++ + void Flusherxxx::Flush(size_t key) { + BatchedEventsList res; + mBatcher.FlushQueue(key, res); + SerializeAndPush(std::move(res)); + } + ``` + + f. `FlushAll`函数： + + ```c++ + void Flusherxxx::FlushAll() { + vector res; + mBatcher.FlushAll(res); + SerializeAndPush(std::move(res)); + } + + ``` + + g. `BuildRequest`函数：将待发送数据包装成一个Http请求，如果请求构建失败，使用`keepItem`参数记录是否要保留数据供以后重试。 + + h. `OnSendDone`函数：根据返回的http response进行相应的记录和操作。 + +6. 在`PluginRegistry`类中注册该插件： + +7. 在pipeline/plugin/PluginRegistry.cpp文件的头文件包含区新增如下行： + + ```c++ + #include "plugin/flusher/Flusherxxx.h" + ``` + +8. 在`PluginRegistry`类的`LoadStaticPlugins()`函数中新增如下行： + + ```c++ + RegisterFlusherCreator(new StaticFlusherCreator()); + ``` diff --git a/docs/cn/developer-guide/plugin-development/how-to-write-native-input-plugins.md b/docs/cn/developer-guide/plugin-development/how-to-write-native-input-plugins.md new file mode 100644 index 0000000000..505cb40509 --- /dev/null +++ b/docs/cn/developer-guide/plugin-development/how-to-write-native-input-plugins.md @@ -0,0 +1,98 @@ +# 如何开发原生Input插件 + +## 工作模式 + +同一输入类型的所有插件实例共享同一个线程来获取数据，插件实例只负责保存插件配置。 + +## 接口定义 + +```c++ +class Input : public Plugin { +public: + // 初始化插件，入参为插件参数 + virtual bool Init(const Json::Value& config) = 0; + // 负责向管理类注册配置 + virtual bool Start() = 0; + // 负责向管理类注销配置 + virtual bool Stop(bool isPipelineRemoving) = 0; +}; +``` + +## 开发步骤 + +1. 在plugin/input目录下新建一个Inputxxx.h和Inputxxx.cpp文件，用于派生Input接口生成具体的插件类； + +2. 在Inputxxx.h文件中定义新的输入插件类Inputxxx，满足以下规范： + + a. 所有的可配置参数的权限为public，其余参数的权限均为private。 + +3. 在Inputxxx.cpp文件中实现`Init`函数，即根据入参初始化插件，针对非法参数，根据非法程度和影响决定是跳过该参数、使用默认值或直接拒绝加载插件。 + +4. 在根目录下新增一个目录，用于创建当前输入插件的管理类及其他辅助类，该管理类需要继承InputRunner接口： + +```c++ +class InputRunner { +public: + // 调用点：由插件的Start函数调用 + // 作用：初始化管理类，并至少启动一个线程用于采集数据 + // 注意：该函数必须是可重入的，因此需要在函数开头判断是否已经启动线程，如果是则直接退出 + virtual void Init() = 0; + // 调用点：进程退出时，或配置热加载结束后无注册插件时由框架调用 + // 作用：停止管理类，并进行扫尾工作，如资源回收、checkpoint记录等 + virtual void Stop() = 0; + // 调用点：每次配置热加载结束后由框架调用 + // 作用：判断是否有插件注册，若无，则框架将调用Stop函数对线程资源进行回收 + virtual bool HasRegisteredPlugin() const = 0; +} +``` + +管理类是输入插件线程资源的实际拥有者，其最基本的运行流程如下： + +- 依次访问每个注册的配置，根据配置情况抓取数据； + +- 根据数据类型将源数据转换为PipelineEvent子类中的一种，并将一批数据组装成PipelineEventGroup； + +- 将PipelineEventGroup发送到相应配置的处理队列中： + +```c++ +ProcessorRunner::GetInstance()->PushQueue(queueKey, inputIdx, std::move(group)); +``` + +其中， + +- queueKey是队列的key，可以从相应流水线的PipelineContext类的`GetProcessQueueKey()`方法来获取。 + +- inputIdx是当前数据所属输入插件在该流水线所有输入插件的位置（即配置中第几个，从0开始计数） + +- group是待发送的数据包 + +最后，为了支持插件向管理类注册，管理类还需要提供注册和注销函数供插件使用，从性能的角度考虑，**该注册和注销过程应当是独立的，即某个插件的注册和注销不应当影响整个线程的运转**。 + +5. 在Inputxxx.cpp文件中实现其余接口函数： + + ```c++ + bool Inputxxx::Start() { + // 1. 调用管理类的Start函数 + // 2. 将当前插件注册到管理类中 + } + + bool Inputxxx::Stop(bool isPipelineRemoving) { + // 将当前插件从管理类中注销 + } + ``` + +6. 在`PluginRegistry`类中注册该插件： + + a. 在pipeline/plugin/PluginRegistry.cpp文件的头文件包含区新增如下行： + + ```c++ + #include "plugin/input/Inputxxx.h" + ``` + + b. 在`PluginRegistry`类的`LoadStaticPlugins()`函数中新增如下行： + + ```c++ + RegisterInputCreator(new StaticInputCreator()); + ``` + + c. 在`PipelineManager`类的构造函数中注册该插件的管理类 From ae008fc0c9ec5ef8edd30e1fab3e7a6916e959d0 Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Fri, 22 Nov 2024 16:46:07 +0800 Subject: [PATCH 08/11] rename pipeline_config to continuous_pipeline_config and rename command to onetime_pipeline_config (#1902) --- config_server/protocol/v2/README.md | 148 +++++++++-------- config_server/protocol/v2/agentV2.proto | 153 ++++++++---------- core/app_config/AppConfig.cpp | 4 +- core/app_config/AppConfig.h | 3 +- core/application/Application.cpp | 6 +- .../common_provider/CommonConfigProvider.cpp | 109 ++++++------- .../common_provider/CommonConfigProvider.h | 22 +-- .../LegacyCommonConfigProvider.cpp | 17 +- .../feedbacker/ConfigFeedbackReceiver.cpp | 44 ++--- .../feedbacker/ConfigFeedbackReceiver.h | 20 +-- core/config/feedbacker/ConfigFeedbackable.h | 4 +- core/config/provider/ConfigProvider.cpp | 10 +- core/config/provider/ConfigProvider.h | 4 +- core/pipeline/PipelineManager.cpp | 19 ++- core/task_pipeline/TaskPipelineManager.cpp | 17 +- .../config/CommonConfigProviderUnittest.cpp | 36 ++--- core/unittest/config/ConfigUpdateUnittest.cpp | 2 +- .../unittest/config/ConfigWatcherUnittest.cpp | 8 +- scripts/dist.sh | 2 +- scripts/gen_build_scripts.sh | 2 +- .../docker-compose.yaml | 2 +- .../docker-compose.yaml | 2 +- test/engine/setup/dockercompose/compose.go | 2 +- 23 files changed, 305 insertions(+), 331 deletions(-) diff --git a/config_server/protocol/v2/README.md b/config_server/protocol/v2/README.md index 53e096f2bd..324f2f8523 100644 --- a/config_server/protocol/v2/README.md +++ b/config_server/protocol/v2/README.md @@ -13,19 +13,19 @@ message HeartbeatRequest { bytes request_id = 1; - uint64 sequence_num = 2; // Increment every request, for server to check sync status - uint64 capabilities = 3; // Bitmask of flags defined by AgentCapabilities enum - bytes instance_id = 4; // Required, Agent's unique identification, consistent throughout the process lifecycle - string agent_type = 5; // Required, Agent's type(ilogtail, ..) - AgentAttributes attributes = 6; // Agent's basic attributes - repeated AgentGroupTag tags = 7; // Agent's tags - string running_status = 8; // Human readable running status - int64 startup_time = 9; // Required, Agent's startup time - repeated ConfigInfo pipeline_configs = 10; // Information about the current PIPELINE_CONFIG held by the Agent - repeated ConfigInfo instance_configs = 11; // Information about the current AGENT_CONFIG held by the Agent - repeated CommandInfo custom_commands = 12; // Information about command history - uint64 flags = 13; // Predefined command flag - bytes opaque = 14; // Opaque data for extension + uint64 sequence_num = 2; // Increment every request, for server to check sync status + uint64 capabilities = 3; // Bitmask of flags defined by AgentCapabilities enum + bytes instance_id = 4; // Required, Agent's unique identification, consistent throughout the process lifecycle + string agent_type = 5; // Required, Agent's type(ilogtail, ..) + AgentAttributes attributes = 6; // Agent's basic attributes + repeated AgentGroupTag tags = 7; // Agent's tags + string running_status = 8; // Human readable running status + int64 startup_time = 9; // Required, Agent's startup time + repeated ConfigInfo continuous_pipeline_configs = 10; // Information about the current continuous pipeline configs held by the Agent + repeated ConfigInfo instance_configs = 11; // Information about the current instance configs held by the Agent + repeated ConfigInfo onetime_pipeline_configs = 12; // Information about onetime pipeline configs history + uint64 flags = 13; // Predefined command flag + bytes opaque = 14; // Opaque data for extension // before 100 (inclusive) are reserved for future official fields } @@ -55,15 +55,6 @@ map extra = 5; // Optional extra info } - // Define the Command information carried in the request - message CommandInfo { - string type = 1; // Command's type - string name = 2; // Required, Command's unique identification - ConfigStatus status = 3; // Command's status - string message = 4; // Optional error message - map extra = 5; // Optional extra info - } - // Define Agent's basic attributes message AgentAttributes { bytes version = 1; // Agent's version @@ -76,12 +67,12 @@ enum AgentCapabilities { // The capabilities field is unspecified. UnspecifiedAgentCapability = 0; - // The Agent can accept pipeline configuration from the Server. - AcceptsPipelineConfig = 0x00000001; + // The Agent can accept continuous pipeline configuration from the Server. + AcceptsContinuousPipelineConfig = 0x00000001; // The Agent can accept instance configuration from the Server. - AcceptsInstanceConfig = 0x00000002; - // The Agent can accept custom command from the Server. - AcceptsCustomCommand = 0x00000004; + AcceptsInstanceConfig = 0x00000002; + // The Agent can accept onetime pipeline configuration from the Server. + AcceptsOnetimePipelineConfig = 0x00000004; // bits before 2^16 (inclusive) are reserved for future official fields } @@ -99,15 +90,15 @@ ### HeartbeatResponse 消息 message HeartbeatResponse { - bytes request_id = 1; - ServerErrorResponse error_response = 2; // Set value indicates error - uint64 capabilities = 3; // Bitmask of flags defined by ServerCapabilities enum - - repeated ConfigDetail pipeline_config_updates = 4; // Agent's pipeline config update status - repeated ConfigDetail instance_config_updates = 5; // Agent's instance config update status - repeated CommandDetail custom_command_updates = 6; // Agent's commands updates - uint64 flags = 7; // Predefined command flag - bytes opaque = 8; // Opaque data for extension + bytes request_id = 1; + CommonResponse common_response = 2; // Set common response + uint64 capabilities = 3; // Bitmask of flags defined by ServerCapabilities enum + + repeated ConfigDetail continuous_pipeline_config_updates = 4; // Agent's continuous pipeline config update status + repeated ConfigDetail instance_config_updates = 5; // Agent's instance config update status + repeated CommandDetail onetime_pipeline_config_updates = 6; // Agent's onetime pipeline config updates + uint64 flags = 7; // Predefined command flag + bytes opaque = 8; // Opaque data for extension } message ConfigDetail { @@ -118,24 +109,23 @@ } message CommandDetail { - string type = 1; // Required, Command type - string name = 2; // Required, Command name - bytes detail = 3; // Required, Command's detail - int64 expire_time = 4; // After which the command can be safely removed from history - map extra = 5; // Optional extra info + string name = 1; // Required, Command name + bytes detail = 2; // Required, Command's detail + int64 expire_time = 3; // After which the command can be safely removed from history + map extra = 4; // Optional extra info } enum ServerCapabilities { // The capabilities field is unspecified. - UnspecifiedServerCapability = 0; + UnspecifiedServerCapability = 0; // The Server can remember agent attributes. - RembersAttribute = 0x00000001; - // The Server can remember pipeline config status. - RembersPipelineConfigStatus = 0x00000002; + RembersAttribute = 0x00000001; + // The Server can remember continuous pipeline config status. + RembersContinuousPipelineConfigStatus = 0x00000002; // The Server can remember instance config status. - RembersInstanceConfigStatus = 0x00000004; - // The Server can remember custom command status. - RembersCustomCommandStatus = 0x00000008; + RembersInstanceConfigStatus = 0x00000004; + // The Server can remember onetime pipeline config status. + RembersOnetimePipelineConfigStatus = 0x00000008; // bits before 2^16 (inclusive) are reserved for future official fields } @@ -154,11 +144,11 @@ // some sub-message in the last AgentToServer message (which is an allowed // optimization) but the Server detects that it does not have it (e.g. was // restarted and lost state). - ReportFullState = 0x00000001; - // FetchPipelineConfigDetail can be used by the Server to tell Agent to fetch config details by FetchConfig api, + ReportFullState = 0x00000001; + // FetchContinuousPipelineConfigDetail can be used by the Server to tell Agent to fetch continuous pipeline config details by FetchConfig api, // HB response ConfigDetail will not contains details. - FetchPipelineConfigDetail = 0x00000002; - // like FetchPipelineConfigDetail, but for instance config. + FetchContinuousPipelineConfigDetail = 0x00000002; + // like FetchContinuousPipelineConfigDetail, but for instance config. FetchInstanceConfigDetail = 0x00000004; // bits before 2^16 (inclusive) are reserved for future official fields } @@ -168,21 +158,21 @@ 额外的 config 拉取接口，不通过心跳返回 config 详情。 message FetchConfigRequest { - bytes request_id = 1; - bytes instance_id = 2; // Agent's unique identification - repeated ConfigInfo pipeline_configs = 3; // Information about the current PIPELINE_CONFIG held by the Agent - repeated ConfigInfo instance_configs = 4; // Information about the current AGENT_CONFIG held by the Agent - repeated CommandInfo custom_commands = 5; // Information about command history + bytes request_id = 1; + bytes instance_id = 2; // Agent's unique identification + repeated ConfigInfo continuous_pipeline_configs = 3; // Information about the current continuous pipeline configs held by the Agent + repeated ConfigInfo instance_configs = 4; // Information about the current instance configs held by the Agent + repeated ConfigInfo onetime_pipeline_configs = 5; // Information about onetime pipeline configs history } ### [Optional] FetchConfigResponse 消息 message FetchConfigResponse { - bytes request_id = 1; - CommonResponse commonResponse = 2; - repeated ConfigDetail pipeline_config_updates = 3; // Agent's pipeline config with details - repeated ConfigDetail instance_config_updates = 4; // Agent's instance config with details - repeated CommandDetail custom_command_updates = 5; // Agent's commands details + bytes request_id = 1; + CommonResponse common_response = 2; + repeated ConfigDetail continuous_pipeline_config_updates = 3; // Agent's continuous pipeline config with details + repeated ConfigDetail instance_config_updates = 4; // Agent's instance config with details + repeated CommandDetail onetime_pipeline_config_updates = 5; // Agent's onetime pipeline config details } ### [Optional] ReportStatusRequest 消息 @@ -191,17 +181,17 @@ message ReportStatusRequest { bytes request_id = 1; - bytes instance_id = 2; // Agent's unique identification - repeated ConfigInfo pipeline_configs = 3; // status about the current PIPELINE_CONFIG held by the Agent - repeated ConfigInfo instance_configs = 4; // status about the current AGENT_CONFIG held by the Agent - repeated CommandInfo custom_commands = 5; // status about command history + bytes instance_id = 2; // Agent's unique identification + repeated ConfigInfo continuous_pipeline_configs = 3; // status about the current continuous pipeline configs held by the Agent + repeated ConfigInfo instance_configs = 4; // status about the current instance configs held by the Agent + repeated ConfigInfo onetime_pipeline_configs = 5; // status about onetime pipeline configs history } ### [Optional] ReportStatusResponse 消息 message ReportStatusResponse { bytes request_id = 1; - CommonResponse commonResponse = 2; + CommonResponse common_response = 2; } ## 行为规范 @@ -218,7 +208,7 @@ Server：应当通过capbilitiies上报Server自身的能力，这样如果新 Client：Agent启动后第一次向Server汇报全量信息，request字段应填尽填。request\_id、sequence\_num、capabilities、instance\_id、agent\_type、startup\_time为必填字段。 -Server：Server根据上报的信息返回响应。pipeline\_config\_updates、instance\_config\_updates中包含agent需要同步的配置，updates中必然包含name和version，是否包含detail取决于server端实现, 如果不包含则需要通过 FetchConfig 拉取。custom\_command_updates包含要求agent执行的命令command中必然包含type、name和expire\_time。 +Server：Server根据上报的信息返回响应。continuous\_pipeline\_config\_updates、instance\_config\_updates中包含agent需要同步的配置，updates中必然包含name和version，是否包含detail取决于server端实现, 如果不包含则需要通过 FetchConfig 拉取。onetime\_pipeline\_config\_updates包含要求agent执行的命令中必然包含name和expire\_time。 Server是否保存Client信息取决于Server实现，如果服务端找不到或保存的sequence\_num + 1 ≠ 心跳的sequence\_num，那么就立刻返回并且flags中必须设置ReportFullStatus标识位。 @@ -243,10 +233,11 @@ Server：同注册 ### 进程配置 可选两种实现： + 1. 在心跳中完成进程配置的状态上报与同步。 Server的注册/心跳响应中有instance\_config\_updates.detail，client 直接从response中获得detail，应用成功后下次心跳需要上报完整状态。 - + 2. 在心跳中完成进程配置的基础信息同步，通过额外的接口完成进程配置的拉取。 Server的响应不包含detail, 只包含要更新的进程配置 name 和 version。client 比较本地的配置和 version 判断需要更新后，根据 instance_config_updates 的信息构造 FetchConfigRequest 后进行一次额外拉取。FetchConfigRequest 至少需要包括 name 和 version。 @@ -258,23 +249,24 @@ Client获取到多个进程配置时，自动合并，若产生冲突默认行 ### 采集配置 可选两种实现： + 1. 在心跳中完成采集配置的状态上报与同步。 - Server的注册/心跳响应中有pipeline\_config\_updates.detail, Client 直接从response中获得detail，应用成功后下次心跳需要上报完整状态。 + Server的注册/心跳响应中有continuous\_pipeline\_config\_updates.detail, Client 直接从response中获得detail，应用成功后下次心跳需要上报完整状态。 2. 在心跳中完成采集配置的基础信息同步，通过额外的接口完成进程配置的拉取。 - Server的响应不包含detail, 只包含要更新的采集配置 name 和 version。client 比较本地的配置和 version 判断需要更新后，根据 pipeline_config_updates 的信息构造 FetchConfigRequest 后进行一次额外拉取。FetchConfigRequest 至少需要包括 name 和 version。 + Server的响应不包含detail, 只包含要更新的采集配置 name 和 version。client 比较本地的配置和 version 判断需要更新后，根据 continuous_pipeline_config_updates 的信息构造 FetchConfigRequest 后进行一次额外拉取。FetchConfigRequest 至少需要包括 name 和 version。 - 心跳 response flag 需要设置 FetchPipelineConfigDetail. + 心跳 response flag 需要设置 FetchContinuousPipelineConfigDetail. 客户端以下2种实现 -实现1：直接将Detail返回在心跳响应中（FetchPipelineConfigDetail flag is unset） +实现1：直接将Detail返回在心跳响应中（FetchContinuousPipelineConfigDetail flag is unset） ![image](https://github.com/alibaba/ilogtail/assets/1827594/be645615-dd99-42dd-9deb-681e9a4069bb) -实现2：仅返回配置名和版本，Detail使用单独请求获取（FetchPipelineConfigDetail flag is set） +实现2：仅返回配置名和版本，Detail使用单独请求获取（FetchContinuousPipelineConfigDetail flag is set） ![image](https://github.com/alibaba/ilogtail/assets/1827594/c409c35c-2a81-4927-bfd2-7fb321ef1ca8) @@ -287,6 +279,7 @@ Client获取到多个进程配置时，自动合并，若产生冲突默认行 对于 Server：这些信息是Agent状态的一部分，可选保存。与通过Event上报可观测信息不同的是，作为状态信息没有时间属性，用户可通过接口可获取即刻状态，而不需要选择时间窗口合并事件。 同进程配置和采集配置，上报配置状态也有两种可选实现： + 1. 在心跳 request 中将配置最新状态带上。 在心跳中将进程配置和采集配置的最新版本和状态一起上报。另外按照心跳协议的定义，配置状态变更后，要求在心跳一定要上报配置最新状态，如果相较于上一次心跳配置状态无变化，则不要求。 @@ -296,7 +289,9 @@ Client获取到多个进程配置时，自动合并，若产生冲突默认行 通过 ReportStatus 额外接口去上报，能够在一定程度上减少心跳服务的复杂度，有利于状态服务和心跳服务的拆分。ReportStatus 接口不用等到下一次心跳，在配置状态发生变化即可上报。 ### 心跳配置拉取/上报与额外接口拉取/上报选择 + 配置状态上报的方式应该和配置拉取方式配套使用： + 1. 如果进程配置和采集配置都通过心跳下发，状态配置也仅应该通过心跳上报。 2. 如果进程配置和采集配置都通过 FetchConfig 接口拉取，状态上报也应该通过 ReportStatus 上报。 @@ -315,7 +310,7 @@ Server: 通过response的flag传递，定义了ReportFullStatus，表明要求C Client: 为了防止服务端重复下发命令以及感知命令执行结果，在command expire前，Client始终应具备向服务端上报command执行状态的能力，实际是否上报取决于心跳压缩机制。在expire\_time超过后，client不应该再上报超时的command状态。 -Server: 如果上报+已知的Agent状态中，缺少应下发的custom\_command\_updates（通过name识别），那么server应该在响应中下发缺少的custom\_command\_updates。 +Server: 如果上报+已知的Agent状态中，缺少应下发的onetime\_pipeline\_config\_updates（通过name识别），那么server应该在响应中下发缺少的onetime\_pipeline\_config\_updates。 ### 异常处理 @@ -324,6 +319,7 @@ Server: 服务端正常返回时HeartbeatResponse中的code应始终设置为0 Client: 当HeartbeatResponse中的code为0时，Agent应该正常处理下发的配置。当HeartbeatResponse中的code不为0时，Agent必须忽略除code和message外的其他字段，并择机重试。 ### 辅助信息 -在command\_info, command\_detail, config\_info, config\_detail中，都预留了extra字段，可以用于传递一些额外的用户自定义的辅助信息。\ + +在command\_detail, config\_info, config\_detail中，都预留了extra字段，可以用于传递一些额外的用户自定义的辅助信息。\ 注意：extra字段仅作传递辅助信息使用，不会对管控行为造成任何影响。 diff --git a/config_server/protocol/v2/agentV2.proto b/config_server/protocol/v2/agentV2.proto index 649cd433da..965e553772 100644 --- a/config_server/protocol/v2/agentV2.proto +++ b/config_server/protocol/v2/agentV2.proto @@ -21,41 +21,32 @@ enum ConfigStatus { // Define the Config information carried in the request message ConfigInfo { - string name = 1; // Required, Config's unique identification - int64 version = 2; // Required, Config's version number or hash code - ConfigStatus status = 3; // Config's status - string message = 4; // Optional error message - map extra = 5; // Optional extra info -} - -// Define the Command information carried in the request -message CommandInfo { - string type = 1; // Command's type - string name = 2; // Required, Command's unique identification - ConfigStatus status = 3; // Command's status - string message = 4; // Optional error message + string name = 1; // Required, Config's unique identification + int64 version = 2; // Required, Config's version number or hash code + ConfigStatus status = 3; // Config's status + string message = 4; // Optional error message map extra = 5; // Optional extra info } // Define Agent's basic attributes message AgentAttributes { - bytes version = 1; // Agent's version - bytes ip = 2; // Agent's ip - bytes hostname = 3; // Agent's hostname - bytes hostid = 4; // Agent's hostid https://opentelemetry.io/docs/specs/semconv/attributes-registry/host/ - map extras = 100; // Agent's other attributes + bytes version = 1; // Agent's version + bytes ip = 2; // Agent's ip + bytes hostname = 3; // Agent's hostname + bytes hostid = 4; // Agent's hostid https://opentelemetry.io/docs/specs/semconv/attributes-registry/host/ + map extras = 100; // Agent's other attributes // before 100 (inclusive) are reserved for future official fields } enum AgentCapabilities { // The capabilities field is unspecified. - UnspecifiedAgentCapability = 0; - // The Agent can accept pipeline configuration from the Server. - AcceptsPipelineConfig = 0x00000001; + UnspecifiedAgentCapability = 0; + // The Agent can accept continuous pipeline configuration from the Server. + AcceptsContinuousPipelineConfig = 0x00000001; // The Agent can accept instance configuration from the Server. - AcceptsInstanceConfig = 0x00000002; - // The Agent can accept custom command from the Server. - AcceptsCustomCommand = 0x00000004; + AcceptsInstanceConfig = 0x00000002; + // The Agent can accept onetime pipeline configuration from the Server. + AcceptsOnetimePipelineConfig = 0x00000004; // Add new capabilities here, continuing with the least significant unused bit. } @@ -66,7 +57,7 @@ enum RequestFlags { // Flags is a bit mask. Values below define individual bits. // Must be set if this request contains full state - FullState = 0x00000001; + FullState = 0x00000001; // bits before 2^16 (inclusive) are reserved for future official fields } @@ -75,49 +66,48 @@ enum RequestFlags { // Agent sends requests to the ConfigServer to get config updates and receive commands. message HeartbeatRequest { bytes request_id = 1; - uint64 sequence_num = 2; // Increment every request, for server to check sync status - uint64 capabilities = 3; // Bitmask of flags defined by AgentCapabilities enum - bytes instance_id = 4; // Required, Agent's unique identification, consistent throughout the process lifecycle - string agent_type = 5; // Required, Agent's type(ilogtail, ..) - AgentAttributes attributes = 6; // Agent's basic attributes - repeated AgentGroupTag tags = 7; // Agent's tags - string running_status = 8; // Human readable running status - int64 startup_time = 9; // Required, Agent's startup time - repeated ConfigInfo pipeline_configs = 10; // Information about the current PIPELINE_CONFIG held by the Agent - repeated ConfigInfo instance_configs = 11; // Information about the current AGENT_CONFIG held by the Agent - repeated CommandInfo custom_commands = 12; // Information about command history - uint64 flags = 13; // Predefined command flag - bytes opaque = 14; // Opaque data for extension + uint64 sequence_num = 2; // Increment every request, for server to check sync status + uint64 capabilities = 3; // Bitmask of flags defined by AgentCapabilities enum + bytes instance_id = 4; // Required, Agent's unique identification, consistent throughout the process lifecycle + string agent_type = 5; // Required, Agent's type(ilogtail, ..) + AgentAttributes attributes = 6; // Agent's basic attributes + repeated AgentGroupTag tags = 7; // Agent's tags + string running_status = 8; // Human readable running status + int64 startup_time = 9; // Required, Agent's startup time + repeated ConfigInfo continuous_pipeline_configs = 10; // Information about the current continuous pipeline configs held by the Agent + repeated ConfigInfo instance_configs = 11; // Information about the current instance configs held by the Agent + repeated ConfigInfo onetime_pipeline_configs = 12; // Information about onetime pipeline configs history + uint64 flags = 13; // Predefined command flag + bytes opaque = 14; // Opaque data for extension // before 100 (inclusive) are reserved for future official fields } // Define Config's detail message ConfigDetail { - string name = 1; // Required, Config's unique identification - int64 version = 2; // Required, Config's version number or hash code - bytes detail = 3; // Required, Config's detail + string name = 1; // Required, Config's unique identification + int64 version = 2; // Required, Config's version number or hash code + bytes detail = 3; // Required, Config's detail map extra = 4; // Optional extra info } message CommandDetail { - string type = 1; // Required, Command type - string name = 2; // Required, Command name - bytes detail = 3; // Required, Command's detail - int64 expire_time = 4; // After which the command can be safely removed from history - map extra = 5; // Optional extra info + string name = 1; // Required, Command name + bytes detail = 2; // Required, Command's detail + int64 expire_time = 3; // After which the command can be safely removed from history + map extra = 4; // Optional extra info } enum ServerCapabilities { // The capabilities field is unspecified. - UnspecifiedServerCapability = 0; + UnspecifiedServerCapability = 0; // The Server can remember agent attributes. - RembersAttribute = 0x00000001; - // The Server can remember pipeline config status. - RembersPipelineConfigStatus = 0x00000002; + RembersAttribute = 0x00000001; + // The Server can remember continuous pipeline config status. + RembersContinuousPipelineConfigStatus = 0x00000002; // The Server can remember instance config status. - RembersInstanceConfigStatus = 0x00000004; - // The Server can remember custom command status. - RembersCustomCommandStatus = 0x00000008; + RembersInstanceConfigStatus = 0x00000004; + // The Server can remember onetime pipeline config status. + RembersOnetimePipelineConfigStatus = 0x00000008; // bits before 2^16 (inclusive) are reserved for future official fields } @@ -131,45 +121,45 @@ enum ResponseFlags { // some sub-message in the last AgentToServer message (which is an allowed // optimization) but the Server detects that it does not have it (e.g. was // restarted and lost state). - ReportFullState = 0x00000001; - // FetchPipelineConfigDetail can be used by the Server to tell Agent to fetch config details by FetchConfig api, + ReportFullState = 0x00000001; + // FetchContinuousPipelineConfigDetail can be used by the Server to tell Agent to fetch continuous pipeline config details by FetchConfig api, // HB response ConfigDetail will not contains details. - FetchPipelineConfigDetail = 0x00000002; - // like FetchPipelineConfigDetail, but for instance config. + FetchContinuousPipelineConfigDetail = 0x00000002; + // like FetchContinuousPipelineConfigDetail, but for instance config. FetchInstanceConfigDetail = 0x00000004; // bits before 2^16 (inclusive) are reserved for future official fields } // ConfigServer's response to Agent's request message HeartbeatResponse { - bytes request_id = 1; - CommonResponse commonResponse = 2; // Set common response - uint64 capabilities = 3; // Bitmask of flags defined by ServerCapabilities enum + bytes request_id = 1; + CommonResponse common_response = 2; // Set common response + uint64 capabilities = 3; // Bitmask of flags defined by ServerCapabilities enum - repeated ConfigDetail pipeline_config_updates = 4; // Agent's pipeline config update status - repeated ConfigDetail instance_config_updates = 5; // Agent's instance config update status - repeated CommandDetail custom_command_updates = 6; // Agent's commands updates - uint64 flags = 7; // Predefined command flag - bytes opaque = 8; // Opaque data for extension + repeated ConfigDetail continuous_pipeline_config_updates = 4; // Agent's continuous pipeline config update status + repeated ConfigDetail instance_config_updates = 5; // Agent's instance config update status + repeated CommandDetail onetime_pipeline_config_updates = 6; // Agent's onetime pipeline config updates + uint64 flags = 7; // Predefined command flag + bytes opaque = 8; // Opaque data for extension } // API: /Agent/FetchConfig // optional api for fetching configs details, but not by heartbeat response with config details, see README. message FetchConfigRequest { - bytes request_id = 1; - bytes instance_id = 2; // Agent's unique identification - repeated ConfigInfo pipeline_configs = 3; // Information about the current PIPELINE_CONFIG held by the Agent - repeated ConfigInfo instance_configs = 4; // Information about the current AGENT_CONFIG held by the Agent - repeated CommandInfo custom_commands = 5; // Information about command history + bytes request_id = 1; + bytes instance_id = 2; // Agent's unique identification + repeated ConfigInfo continuous_pipeline_configs = 3; // Information about the current continuous pipeline configs held by the Agent + repeated ConfigInfo instance_configs = 4; // Information about the current instance configs held by the Agent + repeated ConfigInfo onetime_pipeline_configs = 5; // Information about onetime pipeline configs history } // ConfigServer response to Agent's config fetching request message FetchConfigResponse { - bytes request_id = 1; - CommonResponse commonResponse = 2; - repeated ConfigDetail pipeline_config_updates = 3; // Agent's pipeline config with details - repeated ConfigDetail instance_config_updates = 4; // Agent's instance config with details - repeated CommandDetail custom_command_updates = 5; // Agent's commands details + bytes request_id = 1; + CommonResponse common_response = 2; + repeated ConfigDetail continuous_pipeline_config_updates = 3; // Agent's continuous pipeline config with details + repeated ConfigDetail instance_config_updates = 4; // Agent's instance config with details + repeated CommandDetail onetime_pipeline_config_updates = 5; // Agent's onetime pipeline config details } // API: /Agent/ReportStatus @@ -177,20 +167,19 @@ message FetchConfigResponse { // if HB server and Status server are different service, this api may be help. message ReportStatusRequest { bytes request_id = 1; - bytes instance_id = 2; // Agent's unique identification - repeated ConfigInfo pipeline_configs = 3; // status about the current PIPELINE_CONFIG held by the Agent - repeated ConfigInfo instance_configs = 4; // status about the current AGENT_CONFIG held by the Agent - repeated CommandInfo custom_commands = 5; // status about command history + bytes instance_id = 2; // Agent's unique identification + repeated ConfigInfo continuous_pipeline_configs = 3; // status about the current continuous pipeline configs held by the Agent + repeated ConfigInfo instance_configs = 4; // status about the current instance configs held by the Agent + repeated ConfigInfo onetime_pipeline_configs = 5; // status about onetime pipeline configs history } // ConfigServer response to Agent's report status request message ReportStatusResponse { bytes request_id = 1; - CommonResponse commonResponse = 2; + CommonResponse common_response = 2; } -message CommonResponse -{ +message CommonResponse { int32 status = 1; bytes errorMessage = 2; } diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index e9ecf50e61..dc9b3820c7 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -483,11 +483,11 @@ string GetFileTagsDir() { } } -string GetPipelineConfigDir() { +string GetContinuousPipelineConfigDir() { if (BOOL_FLAG(logtail_mode)) { return "config"; } else { - return "pipeline_config"; + return "continuous_pipeline_config"; } } diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index e2f9929c38..6a970c921f 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -54,6 +54,7 @@ std::string GetObserverEbpfHostPath(); std::string GetSendBufferFileNamePrefix(); std::string GetLegacyUserLocalConfigFilePath(); std::string GetExactlyOnceCheckpoint(); +std::string GetContinuousPipelineConfigDir(); std::string GetPipelineConfigDir(); std::string GetPluginLogName(); std::string GetVersionTag(); @@ -304,7 +305,7 @@ class AppConfig { public: AppConfig(); - ~AppConfig() {}; + ~AppConfig(){}; void LoadInstanceConfig(const std::map>&); diff --git a/core/application/Application.cpp b/core/application/Application.cpp index d6abc71d96..42c0fbc3f3 100644 --- a/core/application/Application.cpp +++ b/core/application/Application.cpp @@ -212,13 +212,13 @@ void Application::Start() { // GCOVR_EXCL_START { // add local config dir - filesystem::path localConfigPath - = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir()) / GetPipelineConfigDir() / "local"; + filesystem::path localConfigPath = filesystem::path(AppConfig::GetInstance()->GetLoongcollectorConfDir()) + / GetContinuousPipelineConfigDir() / "local"; error_code ec; filesystem::create_directories(localConfigPath, ec); if (ec) { LOG_WARNING(sLogger, - ("failed to create dir for local pipeline_config", + ("failed to create dir for local continuous_pipeline_config", "manual creation may be required")("error code", ec.value())("error msg", ec.message())); } PipelineConfigWatcher::GetInstance()->AddSource(localConfigPath.string()); diff --git a/core/config/common_provider/CommonConfigProvider.cpp b/core/config/common_provider/CommonConfigProvider.cpp index a843521915..d8e168c6c9 100644 --- a/core/config/common_provider/CommonConfigProvider.cpp +++ b/core/config/common_provider/CommonConfigProvider.cpp @@ -131,9 +131,9 @@ void CommonConfigProvider::Stop() { void CommonConfigProvider::LoadConfigFile() { error_code ec; - lock_guard pipelineInfomaplock(mPipelineInfoMapMux); - lock_guard lockPipeline(mPipelineMux); - for (auto const& entry : filesystem::directory_iterator(mPipelineSourceDir, ec)) { + lock_guard pipelineInfomaplock(mContinuousPipelineInfoMapMux); + lock_guard lockPipeline(mContinuousPipelineMux); + for (auto const& entry : filesystem::directory_iterator(mContinuousPipelineConfigDir, ec)) { Json::Value detail; if (LoadConfigDetailFromFile(entry, detail)) { ConfigInfo info; @@ -144,8 +144,8 @@ void CommonConfigProvider::LoadConfigFile() { } info.status = ConfigFeedbackStatus::APPLYING; info.detail = detail.toStyledString(); - mPipelineConfigInfoMap[info.name] = info; - ConfigFeedbackReceiver::GetInstance().RegisterPipelineConfig(info.name, this); + mContinuousPipelineConfigInfoMap[info.name] = info; + ConfigFeedbackReceiver::GetInstance().RegisterContinuousPipelineConfig(info.name, this); } } lock_guard instanceInfomaplock(mInstanceInfoMapMux); @@ -266,7 +266,7 @@ configserver::proto::v2::HeartbeatRequest CommonConfigProvider::PrepareHeartbeat heartbeatReq.set_request_id(requestID); heartbeatReq.set_sequence_num(mSequenceNum); heartbeatReq.set_capabilities(configserver::proto::v2::AcceptsInstanceConfig - | configserver::proto::v2::AcceptsPipelineConfig); + | configserver::proto::v2::AcceptsContinuousPipelineConfig); heartbeatReq.set_instance_id(GetInstanceId()); heartbeatReq.set_agent_type("LoongCollector"); FillAttributes(*heartbeatReq.mutable_attributes()); @@ -279,37 +279,18 @@ configserver::proto::v2::HeartbeatRequest CommonConfigProvider::PrepareHeartbeat heartbeatReq.set_running_status("running"); heartbeatReq.set_startup_time(mStartTime); - lock_guard pipelineinfomaplock(mPipelineInfoMapMux); - for (const auto& configInfo : mPipelineConfigInfoMap) { - addConfigInfoToRequest(configInfo, heartbeatReq.add_pipeline_configs()); + lock_guard pipelineinfomaplock(mContinuousPipelineInfoMapMux); + for (const auto& configInfo : mContinuousPipelineConfigInfoMap) { + addConfigInfoToRequest(configInfo, heartbeatReq.add_continuous_pipeline_configs()); } lock_guard instanceinfomaplock(mInstanceInfoMapMux); for (const auto& configInfo : mInstanceConfigInfoMap) { addConfigInfoToRequest(configInfo, heartbeatReq.add_instance_configs()); } - for (auto& configInfo : mCommandInfoMap) { - configserver::proto::v2::CommandInfo* command = heartbeatReq.add_custom_commands(); - command->set_type(configInfo.second.type); - command->set_name(configInfo.second.name); - command->set_message(configInfo.second.message); - switch (configInfo.second.status) { - case ConfigFeedbackStatus::UNSET: - command->set_status(configserver::proto::v2::ConfigStatus::UNSET); - break; - case ConfigFeedbackStatus::APPLYING: - command->set_status(configserver::proto::v2::ConfigStatus::APPLYING); - break; - case ConfigFeedbackStatus::APPLIED: - command->set_status(configserver::proto::v2::ConfigStatus::APPLIED); - break; - case ConfigFeedbackStatus::FAILED: - command->set_status(configserver::proto::v2::ConfigStatus::FAILED); - break; - case ConfigFeedbackStatus::DELETED: - break; - } - command->set_message(configInfo.second.message); + lock_guard onetimeinfomaplock(mOnetimePipelineInfoMapMux); + for (const auto& configInfo : mOnetimePipelineConfigInfoMap) { + addConfigInfoToRequest(configInfo, heartbeatReq.add_onetime_pipeline_configs()); } return heartbeatReq; } @@ -370,10 +351,10 @@ bool CommonConfigProvider::SendHttpRequest(const string& operation, bool CommonConfigProvider::FetchPipelineConfig( configserver::proto::v2::HeartbeatResponse& heartbeatResponse, ::google::protobuf::RepeatedPtrField< ::configserver::proto::v2::ConfigDetail>& result) { - if (heartbeatResponse.flags() & ::configserver::proto::v2::FetchPipelineConfigDetail) { + if (heartbeatResponse.flags() & ::configserver::proto::v2::FetchContinuousPipelineConfigDetail) { return FetchPipelineConfigFromServer(heartbeatResponse, result); } else { - result.Swap(heartbeatResponse.mutable_pipeline_config_updates()); + result.Swap(heartbeatResponse.mutable_continuous_pipeline_config_updates()); return true; } } @@ -381,7 +362,7 @@ bool CommonConfigProvider::FetchPipelineConfig( bool CommonConfigProvider::FetchInstanceConfig( configserver::proto::v2::HeartbeatResponse& heartbeatResponse, ::google::protobuf::RepeatedPtrField< ::configserver::proto::v2::ConfigDetail>& result) { - if (heartbeatResponse.flags() & ::configserver::proto::v2::FetchPipelineConfigDetail) { + if (heartbeatResponse.flags() & ::configserver::proto::v2::FetchContinuousPipelineConfigDetail) { return FetchInstanceConfigFromServer(heartbeatResponse, result); } else { result.Swap(heartbeatResponse.mutable_instance_config_updates()); @@ -422,7 +403,7 @@ bool CommonConfigProvider::DumpConfigFile(const configserver::proto::v2::ConfigD void CommonConfigProvider::UpdateRemotePipelineConfig( const google::protobuf::RepeatedPtrField& configs) { error_code ec; - const std::filesystem::path& sourceDir = mPipelineSourceDir; + const std::filesystem::path& sourceDir = mContinuousPipelineConfigDir; filesystem::create_directories(sourceDir, ec); if (ec) { StopUsingConfigServer(); @@ -432,27 +413,27 @@ void CommonConfigProvider::UpdateRemotePipelineConfig( return; } - lock_guard lock(mPipelineMux); - lock_guard infomaplock(mPipelineInfoMapMux); + lock_guard lock(mContinuousPipelineMux); + lock_guard infomaplock(mContinuousPipelineInfoMapMux); for (const auto& config : configs) { filesystem::path filePath = sourceDir / (config.name() + ".json"); if (config.version() == -1) { - mPipelineConfigInfoMap.erase(config.name()); + mContinuousPipelineConfigInfoMap.erase(config.name()); filesystem::remove(filePath, ec); - ConfigFeedbackReceiver::GetInstance().UnregisterPipelineConfig(config.name()); + ConfigFeedbackReceiver::GetInstance().UnregisterContinuousPipelineConfig(config.name()); } else { if (!DumpConfigFile(config, sourceDir)) { - mPipelineConfigInfoMap[config.name()] = ConfigInfo{.name = config.name(), - .version = config.version(), - .status = ConfigFeedbackStatus::FAILED, - .detail = config.detail()}; + mContinuousPipelineConfigInfoMap[config.name()] = ConfigInfo{.name = config.name(), + .version = config.version(), + .status = ConfigFeedbackStatus::FAILED, + .detail = config.detail()}; continue; } - mPipelineConfigInfoMap[config.name()] = ConfigInfo{.name = config.name(), - .version = config.version(), - .status = ConfigFeedbackStatus::APPLYING, - .detail = config.detail()}; - ConfigFeedbackReceiver::GetInstance().RegisterPipelineConfig(config.name(), this); + mContinuousPipelineConfigInfoMap[config.name()] = ConfigInfo{.name = config.name(), + .version = config.version(), + .status = ConfigFeedbackStatus::APPLYING, + .detail = config.detail()}; + ConfigFeedbackReceiver::GetInstance().RegisterContinuousPipelineConfig(config.name(), this); } } } @@ -536,8 +517,8 @@ bool CommonConfigProvider::FetchPipelineConfigFromServer( string requestID = CalculateRandomUUID(); fetchConfigRequest.set_request_id(requestID); fetchConfigRequest.set_instance_id(GetInstanceId()); - for (const auto& config : heartbeatResponse.pipeline_config_updates()) { - auto reqConfig = fetchConfigRequest.add_pipeline_configs(); + for (const auto& config : heartbeatResponse.continuous_pipeline_config_updates()) { + auto reqConfig = fetchConfigRequest.add_continuous_pipeline_configs(); reqConfig->set_name(config.name()); reqConfig->set_version(config.version()); } @@ -550,20 +531,22 @@ bool CommonConfigProvider::FetchPipelineConfigFromServer( operation, reqBody, "FetchPipelineConfig", fetchConfigRequest.request_id(), fetchConfigResponse)) { configserver::proto::v2::FetchConfigResponse fetchConfigResponsePb; fetchConfigResponsePb.ParseFromString(fetchConfigResponse); - res.Swap(fetchConfigResponsePb.mutable_pipeline_config_updates()); + res.Swap(fetchConfigResponsePb.mutable_continuous_pipeline_config_updates()); return true; } return false; } -void CommonConfigProvider::FeedbackPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status) { - lock_guard infomaplock(mPipelineInfoMapMux); - auto info = mPipelineConfigInfoMap.find(name); - if (info != mPipelineConfigInfoMap.end()) { +void CommonConfigProvider::FeedbackContinuousPipelineConfigStatus(const std::string& name, + ConfigFeedbackStatus status) { + lock_guard infomaplock(mContinuousPipelineInfoMapMux); + auto info = mContinuousPipelineConfigInfoMap.find(name); + if (info != mContinuousPipelineConfigInfoMap.end()) { info->second.status = status; } LOG_DEBUG(sLogger, - ("CommonConfigProvider", "FeedbackPipelineConfigStatus")("name", name)("status", ToStringView(status))); + ("CommonConfigProvider", "FeedbackContinuousPipelineConfigStatus")("name", name)("status", + ToStringView(status))); } void CommonConfigProvider::FeedbackInstanceConfigStatus(const std::string& name, ConfigFeedbackStatus status) { lock_guard infomaplock(mInstanceInfoMapMux); @@ -574,17 +557,17 @@ void CommonConfigProvider::FeedbackInstanceConfigStatus(const std::string& name, LOG_DEBUG(sLogger, ("CommonConfigProvider", "FeedbackInstanceConfigStatus")("name", name)("status", ToStringView(status))); } -void CommonConfigProvider::FeedbackCommandConfigStatus(const std::string& type, - const std::string& name, - ConfigFeedbackStatus status) { - lock_guard infomaplock(mCommondInfoMapMux); - auto info = mCommandInfoMap.find(GenerateCommandFeedBackKey(type, name)); - if (info != mCommandInfoMap.end()) { +void CommonConfigProvider::FeedbackOnetimePipelineConfigStatus(const std::string& type, + const std::string& name, + ConfigFeedbackStatus status) { + lock_guard infomaplock(mOnetimePipelineInfoMapMux); + auto info = mOnetimePipelineConfigInfoMap.find(GenerateOnetimePipelineConfigFeedBackKey(type, name)); + if (info != mOnetimePipelineConfigInfoMap.end()) { info->second.status = status; } LOG_DEBUG(sLogger, ("CommonConfigProvider", - "FeedbackCommandConfigStatus")("type", type)("name", name)("status", ToStringView(status))); + "FeedbackOnetimePipelineConfigStatus")("type", type)("name", name)("status", ToStringView(status))); } } // namespace logtail diff --git a/core/config/common_provider/CommonConfigProvider.h b/core/config/common_provider/CommonConfigProvider.h index d79d22a95e..f7a4e918a6 100644 --- a/core/config/common_provider/CommonConfigProvider.h +++ b/core/config/common_provider/CommonConfigProvider.h @@ -38,13 +38,6 @@ struct ConfigInfo { std::string detail; }; -struct CommandInfo { - std::string type; - std::string name; - ConfigFeedbackStatus status; - std::string message; -}; - class CommonConfigProvider : public ConfigProvider, ConfigFeedbackable { public: std::string sName; @@ -61,10 +54,11 @@ class CommonConfigProvider : public ConfigProvider, ConfigFeedbackable { void Init(const std::string& dir) override; void Stop() override; - void FeedbackPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status) override; + void FeedbackContinuousPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status) override; void FeedbackInstanceConfigStatus(const std::string& name, ConfigFeedbackStatus status) override; - void - FeedbackCommandConfigStatus(const std::string& type, const std::string& name, ConfigFeedbackStatus status) override; + void FeedbackOnetimePipelineConfigStatus(const std::string& type, + const std::string& name, + ConfigFeedbackStatus status) override; CommonConfigProvider() = default; ~CommonConfigProvider() = default; @@ -105,12 +99,12 @@ class CommonConfigProvider : public ConfigProvider, ConfigFeedbackable { bool mConfigServerAvailable = false; mutable std::mutex mInstanceInfoMapMux; - mutable std::mutex mPipelineInfoMapMux; - mutable std::mutex mCommondInfoMapMux; + mutable std::mutex mContinuousPipelineInfoMapMux; + mutable std::mutex mOnetimePipelineInfoMapMux; - std::unordered_map mPipelineConfigInfoMap; + std::unordered_map mContinuousPipelineConfigInfoMap; std::unordered_map mInstanceConfigInfoMap; - std::unordered_map mCommandInfoMap; + std::unordered_map mOnetimePipelineConfigInfoMap; private: static std::string configVersion; diff --git a/core/config/common_provider/LegacyCommonConfigProvider.cpp b/core/config/common_provider/LegacyCommonConfigProvider.cpp index b673bbeda5..4c1b70ea53 100644 --- a/core/config/common_provider/LegacyCommonConfigProvider.cpp +++ b/core/config/common_provider/LegacyCommonConfigProvider.cpp @@ -286,20 +286,21 @@ void LegacyCommonConfigProvider::UpdateRemoteConfig( const google::protobuf::RepeatedPtrField& checkResults, const google::protobuf::RepeatedPtrField& configDetails) { error_code ec; - filesystem::create_directories(mPipelineSourceDir, ec); + filesystem::create_directories(mContinuousPipelineConfigDir, ec); if (ec) { StopUsingConfigServer(); - LOG_ERROR(sLogger, - ("failed to create dir for legacy common configs", - "stop receiving config from legacy common config server")("dir", mPipelineSourceDir.string())( - "error code", ec.value())("error msg", ec.message())); + LOG_ERROR( + sLogger, + ("failed to create dir for legacy common configs", + "stop receiving config from legacy common config server")("dir", mContinuousPipelineConfigDir.string())( + "error code", ec.value())("error msg", ec.message())); return; } - lock_guard lock(mPipelineMux); + lock_guard lock(mContinuousPipelineMux); for (const auto& checkResult : checkResults) { - filesystem::path filePath = mPipelineSourceDir / (checkResult.name() + ".yaml"); - filesystem::path tmpFilePath = mPipelineSourceDir / (checkResult.name() + ".yaml.new"); + filesystem::path filePath = mContinuousPipelineConfigDir / (checkResult.name() + ".yaml"); + filesystem::path tmpFilePath = mContinuousPipelineConfigDir / (checkResult.name() + ".yaml.new"); switch (checkResult.check_status()) { case configserver::proto::DELETED: mConfigNameVersionMap.erase(checkResult.name()); diff --git a/core/config/feedbacker/ConfigFeedbackReceiver.cpp b/core/config/feedbacker/ConfigFeedbackReceiver.cpp index b9807ef9e4..4bad1dabed 100644 --- a/core/config/feedbacker/ConfigFeedbackReceiver.cpp +++ b/core/config/feedbacker/ConfigFeedbackReceiver.cpp @@ -26,9 +26,10 @@ ConfigFeedbackReceiver& ConfigFeedbackReceiver::GetInstance() { return instance; } -void ConfigFeedbackReceiver::RegisterPipelineConfig(const std::string& name, ConfigFeedbackable* feedbackable) { +void ConfigFeedbackReceiver::RegisterContinuousPipelineConfig(const std::string& name, + ConfigFeedbackable* feedbackable) { std::lock_guard lock(mMutex); - mPipelineConfigFeedbackableMap[name] = feedbackable; + mContinuousPipelineConfigFeedbackableMap[name] = feedbackable; } void ConfigFeedbackReceiver::RegisterInstanceConfig(const std::string& name, ConfigFeedbackable* feedbackable) { @@ -36,16 +37,16 @@ void ConfigFeedbackReceiver::RegisterInstanceConfig(const std::string& name, Con mInstanceConfigFeedbackableMap[name] = feedbackable; } -void ConfigFeedbackReceiver::RegisterCommand(const std::string& type, - const std::string& name, - ConfigFeedbackable* feedbackable) { +void ConfigFeedbackReceiver::RegisterOnetimePipelineConfig(const std::string& type, + const std::string& name, + ConfigFeedbackable* feedbackable) { std::lock_guard lock(mMutex); - mCommandFeedbackableMap[GenerateCommandFeedBackKey(type, name)] = feedbackable; + mOnetimePipelineConfigFeedbackableMap[GenerateOnetimePipelineConfigFeedBackKey(type, name)] = feedbackable; } -void ConfigFeedbackReceiver::UnregisterPipelineConfig(const std::string& name) { +void ConfigFeedbackReceiver::UnregisterContinuousPipelineConfig(const std::string& name) { std::lock_guard lock(mMutex); - mPipelineConfigFeedbackableMap.erase(name); + mContinuousPipelineConfigFeedbackableMap.erase(name); } void ConfigFeedbackReceiver::UnregisterInstanceConfig(const std::string& name) { @@ -53,16 +54,17 @@ void ConfigFeedbackReceiver::UnregisterInstanceConfig(const std::string& name) { mInstanceConfigFeedbackableMap.erase(name); } -void ConfigFeedbackReceiver::UnregisterCommand(const std::string& type, const std::string& name) { +void ConfigFeedbackReceiver::UnregisterOnetimePipelineConfig(const std::string& type, const std::string& name) { std::lock_guard lock(mMutex); - mCommandFeedbackableMap.erase(GenerateCommandFeedBackKey(type, name)); + mOnetimePipelineConfigFeedbackableMap.erase(GenerateOnetimePipelineConfigFeedBackKey(type, name)); } -void ConfigFeedbackReceiver::FeedbackPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status) { +void ConfigFeedbackReceiver::FeedbackContinuousPipelineConfigStatus(const std::string& name, + ConfigFeedbackStatus status) { std::lock_guard lock(mMutex); - auto iter = mPipelineConfigFeedbackableMap.find(name); - if (iter != mPipelineConfigFeedbackableMap.end()) { - iter->second->FeedbackPipelineConfigStatus(name, status); + auto iter = mContinuousPipelineConfigFeedbackableMap.find(name); + if (iter != mContinuousPipelineConfigFeedbackableMap.end()) { + iter->second->FeedbackContinuousPipelineConfigStatus(name, status); } } @@ -74,17 +76,17 @@ void ConfigFeedbackReceiver::FeedbackInstanceConfigStatus(const std::string& nam } } -void ConfigFeedbackReceiver::FeedbackCommandConfigStatus(const std::string& type, - const std::string& name, - ConfigFeedbackStatus status) { +void ConfigFeedbackReceiver::FeedbackOnetimePipelineConfigStatus(const std::string& type, + const std::string& name, + ConfigFeedbackStatus status) { std::lock_guard lock(mMutex); - auto iter = mCommandFeedbackableMap.find(GenerateCommandFeedBackKey(type, name)); - if (iter != mCommandFeedbackableMap.end()) { - iter->second->FeedbackCommandConfigStatus(type, name, status); + auto iter = mOnetimePipelineConfigFeedbackableMap.find(GenerateOnetimePipelineConfigFeedBackKey(type, name)); + if (iter != mOnetimePipelineConfigFeedbackableMap.end()) { + iter->second->FeedbackOnetimePipelineConfigStatus(type, name, status); } } -std::string GenerateCommandFeedBackKey(const std::string& type, const std::string& name) { +std::string GenerateOnetimePipelineConfigFeedBackKey(const std::string& type, const std::string& name) { return type + '\1' + name; } diff --git a/core/config/feedbacker/ConfigFeedbackReceiver.h b/core/config/feedbacker/ConfigFeedbackReceiver.h index 86796afbcd..f20bcdb73c 100644 --- a/core/config/feedbacker/ConfigFeedbackReceiver.h +++ b/core/config/feedbacker/ConfigFeedbackReceiver.h @@ -23,27 +23,29 @@ namespace logtail { -std::string GenerateCommandFeedBackKey(const std::string& type, const std::string& name); +std::string GenerateOnetimePipelineConfigFeedBackKey(const std::string& type, const std::string& name); class ConfigFeedbackReceiver { public: static ConfigFeedbackReceiver& GetInstance(); - void RegisterPipelineConfig(const std::string& name, ConfigFeedbackable* feedbackable); + void RegisterContinuousPipelineConfig(const std::string& name, ConfigFeedbackable* feedbackable); void RegisterInstanceConfig(const std::string& name, ConfigFeedbackable* feedbackable); - void RegisterCommand(const std::string& type, const std::string& name, ConfigFeedbackable* feedbackable); - void UnregisterPipelineConfig(const std::string& name); + void + RegisterOnetimePipelineConfig(const std::string& type, const std::string& name, ConfigFeedbackable* feedbackable); + void UnregisterContinuousPipelineConfig(const std::string& name); void UnregisterInstanceConfig(const std::string& name); - void UnregisterCommand(const std::string& type, const std::string& name); - void FeedbackPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status); + void UnregisterOnetimePipelineConfig(const std::string& type, const std::string& name); + void FeedbackContinuousPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status); void FeedbackInstanceConfigStatus(const std::string& name, ConfigFeedbackStatus status); - void FeedbackCommandConfigStatus(const std::string& type, const std::string& name, ConfigFeedbackStatus status); + void + FeedbackOnetimePipelineConfigStatus(const std::string& type, const std::string& name, ConfigFeedbackStatus status); private: ConfigFeedbackReceiver() {} std::mutex mMutex; - std::unordered_map mPipelineConfigFeedbackableMap; + std::unordered_map mContinuousPipelineConfigFeedbackableMap; std::unordered_map mInstanceConfigFeedbackableMap; - std::unordered_map mCommandFeedbackableMap; + std::unordered_map mOnetimePipelineConfigFeedbackableMap; }; } // namespace logtail diff --git a/core/config/feedbacker/ConfigFeedbackable.h b/core/config/feedbacker/ConfigFeedbackable.h index f027e2e758..29126c9d33 100644 --- a/core/config/feedbacker/ConfigFeedbackable.h +++ b/core/config/feedbacker/ConfigFeedbackable.h @@ -28,10 +28,10 @@ std::string_view ToStringView(ConfigFeedbackStatus status); class ConfigFeedbackable { public: virtual ~ConfigFeedbackable() = default; // LCOV_EXCL_LINE - virtual void FeedbackPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status) = 0; + virtual void FeedbackContinuousPipelineConfigStatus(const std::string& name, ConfigFeedbackStatus status) = 0; virtual void FeedbackInstanceConfigStatus(const std::string& name, ConfigFeedbackStatus status) = 0; virtual void - FeedbackCommandConfigStatus(const std::string& type, const std::string& name, ConfigFeedbackStatus status) + FeedbackOnetimePipelineConfigStatus(const std::string& type, const std::string& name, ConfigFeedbackStatus status) = 0; }; diff --git a/core/config/provider/ConfigProvider.cpp b/core/config/provider/ConfigProvider.cpp index 1c6b68b31b..207f52a39a 100644 --- a/core/config/provider/ConfigProvider.cpp +++ b/core/config/provider/ConfigProvider.cpp @@ -24,17 +24,17 @@ namespace logtail { void ConfigProvider::Init(const string& dir) { // default path: /etc/ilogtail/config/${dir} - mPipelineSourceDir.assign(AppConfig::GetInstance()->GetLoongcollectorConfDir()); - mPipelineSourceDir /= GetPipelineConfigDir(); - mPipelineSourceDir /= dir; + mContinuousPipelineConfigDir.assign(AppConfig::GetInstance()->GetLoongcollectorConfDir()); + mContinuousPipelineConfigDir /= GetContinuousPipelineConfigDir(); + mContinuousPipelineConfigDir /= dir; mInstanceSourceDir.assign(AppConfig::GetInstance()->GetLoongcollectorConfDir()); mInstanceSourceDir /= "instance_config"; mInstanceSourceDir /= dir; error_code ec; - filesystem::create_directories(mPipelineSourceDir, ec); - PipelineConfigWatcher::GetInstance()->AddSource(mPipelineSourceDir, &mPipelineMux); + filesystem::create_directories(mContinuousPipelineConfigDir, ec); + PipelineConfigWatcher::GetInstance()->AddSource(mContinuousPipelineConfigDir, &mContinuousPipelineMux); ec.clear(); filesystem::create_directories(mInstanceSourceDir, ec); diff --git a/core/config/provider/ConfigProvider.h b/core/config/provider/ConfigProvider.h index ed58ac3c11..b41f663a8a 100644 --- a/core/config/provider/ConfigProvider.h +++ b/core/config/provider/ConfigProvider.h @@ -34,9 +34,9 @@ class ConfigProvider { ConfigProvider() = default; virtual ~ConfigProvider() = default; - std::filesystem::path mPipelineSourceDir; + std::filesystem::path mContinuousPipelineConfigDir; std::filesystem::path mInstanceSourceDir; - mutable std::mutex mPipelineMux; + mutable std::mutex mContinuousPipelineMux; mutable std::mutex mInstanceMux; }; diff --git a/core/pipeline/PipelineManager.cpp b/core/pipeline/PipelineManager.cpp index 1fe4709564..57abcde874 100644 --- a/core/pipeline/PipelineManager.cpp +++ b/core/pipeline/PipelineManager.cpp @@ -40,7 +40,7 @@ PipelineManager::PipelineManager() : mInputRunners({ PrometheusInputRunner::GetInstance(), #if defined(__linux__) && !defined(__ANDROID__) - ebpf::eBPFServer::GetInstance(), + ebpf::eBPFServer::GetInstance(), #endif }) { } @@ -78,7 +78,8 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { DecreasePluginUsageCnt(iter->second->GetPluginStatistics()); iter->second->RemoveProcessQueue(); mPipelineNameEntityMap.erase(iter); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(name, ConfigFeedbackStatus::DELETED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(name, + ConfigFeedbackStatus::DELETED); } for (auto& config : diff.mModified) { auto p = BuildPipeline(std::move(config)); // auto reuse old pipeline's process queue and sender queue @@ -92,8 +93,8 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { config.mProject, config.mLogstore, config.mRegion); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, - ConfigFeedbackStatus::FAILED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::FAILED); continue; } LOG_INFO(sLogger, @@ -106,7 +107,8 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { mPipelineNameEntityMap[config.mName] = p; IncreasePluginUsageCnt(p->GetPluginStatistics()); p->Start(); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, ConfigFeedbackStatus::APPLIED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::APPLIED); } for (auto& config : diff.mAdded) { auto p = BuildPipeline(std::move(config)); @@ -119,8 +121,8 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { config.mProject, config.mLogstore, config.mRegion); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, - ConfigFeedbackStatus::FAILED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::FAILED); continue; } LOG_INFO(sLogger, @@ -128,7 +130,8 @@ void logtail::PipelineManager::UpdatePipelines(PipelineConfigDiff& diff) { mPipelineNameEntityMap[config.mName] = p; IncreasePluginUsageCnt(p->GetPluginStatistics()); p->Start(); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, ConfigFeedbackStatus::APPLIED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::APPLIED); } #ifndef APSARA_UNIT_TEST_MAIN diff --git a/core/task_pipeline/TaskPipelineManager.cpp b/core/task_pipeline/TaskPipelineManager.cpp index a846f1dd4d..44d9b50195 100644 --- a/core/task_pipeline/TaskPipelineManager.cpp +++ b/core/task_pipeline/TaskPipelineManager.cpp @@ -29,7 +29,8 @@ void TaskPipelineManager::UpdatePipelines(TaskConfigDiff& diff) { auto iter = mPipelineNameEntityMap.find(name); iter->second->Stop(true); mPipelineNameEntityMap.erase(iter); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(name, ConfigFeedbackStatus::DELETED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(name, + ConfigFeedbackStatus::DELETED); } for (auto& config : diff.mModified) { auto p = BuildPipeline(std::move(config)); @@ -40,8 +41,8 @@ void TaskPipelineManager::UpdatePipelines(TaskConfigDiff& diff) { AlarmManager::GetInstance()->SendAlarm( CATEGORY_CONFIG_ALARM, "failed to build task for existing config: keep current task running, config: " + config.mName); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, - ConfigFeedbackStatus::FAILED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::FAILED); continue; } LOG_INFO(sLogger, @@ -51,7 +52,8 @@ void TaskPipelineManager::UpdatePipelines(TaskConfigDiff& diff) { iter->second->Stop(false); mPipelineNameEntityMap[config.mName] = std::move(p); mPipelineNameEntityMap[config.mName]->Start(); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, ConfigFeedbackStatus::APPLIED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::APPLIED); } for (auto& config : diff.mAdded) { auto p = BuildPipeline(std::move(config)); @@ -61,14 +63,15 @@ void TaskPipelineManager::UpdatePipelines(TaskConfigDiff& diff) { AlarmManager::GetInstance()->SendAlarm(CATEGORY_CONFIG_ALARM, "failed to build task for new config: skip current object, config: " + config.mName); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, - ConfigFeedbackStatus::FAILED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::FAILED); continue; } LOG_INFO(sLogger, ("task building for new config succeeded", "begin to start task")("config", config.mName)); mPipelineNameEntityMap[config.mName] = std::move(p); mPipelineNameEntityMap[config.mName]->Start(); - ConfigFeedbackReceiver::GetInstance().FeedbackPipelineConfigStatus(config.mName, ConfigFeedbackStatus::APPLIED); + ConfigFeedbackReceiver::GetInstance().FeedbackContinuousPipelineConfigStatus(config.mName, + ConfigFeedbackStatus::APPLIED); } } diff --git a/core/unittest/config/CommonConfigProviderUnittest.cpp b/core/unittest/config/CommonConfigProviderUnittest.cpp index 1fd43d0652..0621cf0f26 100644 --- a/core/unittest/config/CommonConfigProviderUnittest.cpp +++ b/core/unittest/config/CommonConfigProviderUnittest.cpp @@ -21,8 +21,8 @@ #include "config/ConfigDiff.h" #include "config/InstanceConfigManager.h" #include "config/common_provider/CommonConfigProvider.h" -#include "config/watcher/PipelineConfigWatcher.h" #include "config/watcher/InstanceConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" #include "gmock/gmock.h" #include "monitor/Monitor.h" #include "pipeline/PipelineManager.h" @@ -80,7 +80,7 @@ class CommonConfigProviderUnittest : public ::testing::Test { MockCommonConfigProvider provider; provider.Init("common_v2"); provider.Stop(); - bfs::remove_all(provider.mPipelineSourceDir.string()); + bfs::remove_all(provider.mContinuousPipelineConfigDir.string()); bfs::remove_all(provider.mInstanceSourceDir.string()); } else { CreateAgentDir(); @@ -91,7 +91,7 @@ class CommonConfigProviderUnittest : public ::testing::Test { MockCommonConfigProvider provider; provider.Init("common_v2"); provider.Stop(); - bfs::remove_all(provider.mPipelineSourceDir.string()); + bfs::remove_all(provider.mContinuousPipelineConfigDir.string()); bfs::remove_all(provider.mInstanceSourceDir.string()); } } @@ -101,7 +101,7 @@ class CommonConfigProviderUnittest : public ::testing::Test { MockCommonConfigProvider provider; provider.Init("common_v2"); provider.Stop(); - bfs::remove_all(provider.mPipelineSourceDir.string()); + bfs::remove_all(provider.mContinuousPipelineConfigDir.string()); bfs::remove_all(provider.mInstanceSourceDir.string()); } @@ -294,7 +294,8 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { APSARA_TEST_EQUAL(heartbeatReq.sequence_num(), sequence_num); sequence_num++; APSARA_TEST_TRUE(heartbeatReq.capabilities() & configserver::proto::v2::AcceptsInstanceConfig); - APSARA_TEST_TRUE(heartbeatReq.capabilities() & configserver::proto::v2::AcceptsPipelineConfig); + APSARA_TEST_TRUE(heartbeatReq.capabilities() + & configserver::proto::v2::AcceptsContinuousPipelineConfig); APSARA_TEST_EQUAL(heartbeatReq.instance_id(), provider.GetInstanceId()); APSARA_TEST_EQUAL(heartbeatReq.agent_type(), "LoongCollector"); APSARA_TEST_EQUAL(heartbeatReq.attributes().ip(), LoongCollectorMonitor::mIpAddr); @@ -314,7 +315,7 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { configserver::proto::v2::HeartbeatResponse heartbeatRespPb; heartbeatRespPb.set_capabilities(configserver::proto::v2::ResponseFlags::ReportFullState); { - auto pipeline = heartbeatRespPb.mutable_pipeline_config_updates(); + auto pipeline = heartbeatRespPb.mutable_continuous_pipeline_config_updates(); auto configDetail = pipeline->Add(); configDetail->set_name("config1"); configDetail->set_version(1); @@ -382,10 +383,9 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { )"); } { - auto commandconfig = heartbeatRespPb.mutable_custom_command_updates(); + auto commandconfig = heartbeatRespPb.mutable_onetime_pipeline_config_updates(); auto configDetail = commandconfig->Add(); configDetail->set_name("commandconfig1"); - configDetail->set_type("history"); configDetail->set_detail(R"( { "enable": true, @@ -427,9 +427,9 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { configserver::proto::v2::HeartbeatResponse heartbeatResponse; provider.GetConfigUpdate(); - APSARA_TEST_EQUAL(provider.mPipelineConfigInfoMap.size(), 2); - APSARA_TEST_EQUAL(provider.mPipelineConfigInfoMap["config1"].status, ConfigFeedbackStatus::APPLYING); - APSARA_TEST_EQUAL(provider.mPipelineConfigInfoMap["config2"].status, ConfigFeedbackStatus::FAILED); + APSARA_TEST_EQUAL(provider.mContinuousPipelineConfigInfoMap.size(), 2); + APSARA_TEST_EQUAL(provider.mContinuousPipelineConfigInfoMap["config1"].status, ConfigFeedbackStatus::APPLYING); + APSARA_TEST_EQUAL(provider.mContinuousPipelineConfigInfoMap["config2"].status, ConfigFeedbackStatus::FAILED); // 处理 pipelineconfig auto pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); @@ -474,8 +474,8 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { { MockCommonConfigProvider provider; provider.Init("common_v2"); - APSARA_TEST_EQUAL(provider.mPipelineConfigInfoMap.size(), 1); - APSARA_TEST_EQUAL(provider.mPipelineConfigInfoMap["config1"].status, ConfigFeedbackStatus::APPLYING); + APSARA_TEST_EQUAL(provider.mContinuousPipelineConfigInfoMap.size(), 1); + APSARA_TEST_EQUAL(provider.mContinuousPipelineConfigInfoMap["config1"].status, ConfigFeedbackStatus::APPLYING); APSARA_TEST_EQUAL(provider.mInstanceConfigInfoMap.size(), 1); APSARA_TEST_EQUAL(provider.mInstanceConfigInfoMap["instanceconfig1"].status, ConfigFeedbackStatus::APPLYING); provider.Stop(); @@ -509,7 +509,8 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { APSARA_TEST_EQUAL(heartbeatReq.sequence_num(), sequence_num); sequence_num++; APSARA_TEST_TRUE(heartbeatReq.capabilities() & configserver::proto::v2::AcceptsInstanceConfig); - APSARA_TEST_TRUE(heartbeatReq.capabilities() & configserver::proto::v2::AcceptsPipelineConfig); + APSARA_TEST_TRUE(heartbeatReq.capabilities() + & configserver::proto::v2::AcceptsContinuousPipelineConfig); APSARA_TEST_EQUAL(heartbeatReq.instance_id(), provider.GetInstanceId()); APSARA_TEST_EQUAL(heartbeatReq.agent_type(), "LoongCollector"); APSARA_TEST_EQUAL(heartbeatReq.attributes().ip(), LoongCollectorMonitor::mIpAddr); @@ -530,7 +531,7 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { heartbeatRespPb.set_capabilities(configserver::proto::v2::ResponseFlags::ReportFullState); // pipeline { - auto pipeline = heartbeatRespPb.mutable_pipeline_config_updates(); + auto pipeline = heartbeatRespPb.mutable_continuous_pipeline_config_updates(); auto configDetail = pipeline->Add(); configDetail->set_name("config1"); configDetail->set_version(-1); @@ -600,10 +601,9 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { } // commandconfig { - auto commandconfig = heartbeatRespPb.mutable_custom_command_updates(); + auto commandconfig = heartbeatRespPb.mutable_onetime_pipeline_config_updates(); auto configDetail = commandconfig->Add(); configDetail->set_name("commandconfig1"); - configDetail->set_type("history"); configDetail->set_detail(R"( { "enable": true, @@ -645,7 +645,7 @@ void CommonConfigProviderUnittest::TestGetConfigUpdateAndConfigWatcher() { configserver::proto::v2::HeartbeatResponse heartbeatResponse; provider.GetConfigUpdate(); - APSARA_TEST_TRUE(provider.mPipelineConfigInfoMap.empty()); + APSARA_TEST_TRUE(provider.mContinuousPipelineConfigInfoMap.empty()); // 处理pipelineConfigDiff auto pipelineConfigDiff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); diff --git a/core/unittest/config/ConfigUpdateUnittest.cpp b/core/unittest/config/ConfigUpdateUnittest.cpp index c72f645e1a..86c7c7ae84 100644 --- a/core/unittest/config/ConfigUpdateUnittest.cpp +++ b/core/unittest/config/ConfigUpdateUnittest.cpp @@ -101,7 +101,7 @@ class ConfigUpdateUnittest : public testing::Test { void PrepareInitialSettings() const; void GenerateInitialConfigs() const; - filesystem::path configDir = "./pipeline_config"; + filesystem::path configDir = "./continuous_pipeline_config"; vector pipelineConfigPaths = {configDir / "pipeline_invalid_format.json", configDir / "pipeline_invalid_detail.json", configDir / "pipeline_enabled_valid.json", diff --git a/core/unittest/config/ConfigWatcherUnittest.cpp b/core/unittest/config/ConfigWatcherUnittest.cpp index 4edec0c4f2..7ad1cc1fe0 100644 --- a/core/unittest/config/ConfigWatcherUnittest.cpp +++ b/core/unittest/config/ConfigWatcherUnittest.cpp @@ -16,8 +16,8 @@ #include #include "config/ConfigDiff.h" -#include "config/watcher/PipelineConfigWatcher.h" #include "config/watcher/InstanceConfigWatcher.h" +#include "config/watcher/PipelineConfigWatcher.h" #include "pipeline/plugin/PluginRegistry.h" #include "unittest/Unittest.h" @@ -44,7 +44,7 @@ class ConfigWatcherUnittest : public testing::Test { static const filesystem::path instanceConfigDir; }; -const filesystem::path ConfigWatcherUnittest::configDir = "./pipeline_config"; +const filesystem::path ConfigWatcherUnittest::configDir = "./continuous_pipeline_config"; const filesystem::path ConfigWatcherUnittest::instanceConfigDir = "./instance_config"; void ConfigWatcherUnittest::InvalidConfigDirFound() const { @@ -53,11 +53,11 @@ void ConfigWatcherUnittest::InvalidConfigDirFound() const { APSARA_TEST_TRUE(diff.first.IsEmpty()); APSARA_TEST_TRUE(diff.second.IsEmpty()); - { ofstream fout("pipeline_config"); } + { ofstream fout("continuous_pipeline_config"); } diff = PipelineConfigWatcher::GetInstance()->CheckConfigDiff(); APSARA_TEST_TRUE(diff.first.IsEmpty()); APSARA_TEST_TRUE(diff.second.IsEmpty()); - filesystem::remove_all("pipeline_config"); + filesystem::remove_all("continuous_pipeline_config"); } { InstanceConfigDiff diff = InstanceConfigWatcher::GetInstance()->CheckConfigDiff(); diff --git a/scripts/dist.sh b/scripts/dist.sh index 74fbc777f3..946d6c328f 100755 --- a/scripts/dist.sh +++ b/scripts/dist.sh @@ -42,7 +42,7 @@ cp "${ROOTDIR}/${OUT_DIR}/libGoPluginAdapter.so" "${ROOTDIR}/${DIST_DIR}/${PACKA cp "${ROOTDIR}/${OUT_DIR}/libGoPluginBase.so" "${ROOTDIR}/${DIST_DIR}/${PACKAGE_DIR}" mkdir -p "${ROOTDIR}/${DIST_DIR}/${PACKAGE_DIR}/conf/instance_config/local/" cp "${ROOTDIR}/${OUT_DIR}/conf/instance_config/local/loongcollector_config.json" "${ROOTDIR}/${DIST_DIR}/${PACKAGE_DIR}/conf/instance_config/local/" -cp -a "${ROOTDIR}/${OUT_DIR}/conf/pipeline_config/local" "${ROOTDIR}/${DIST_DIR}/${PACKAGE_DIR}/conf" +cp -a "${ROOTDIR}/${OUT_DIR}/conf/continuous_pipeline_config/local" "${ROOTDIR}/${DIST_DIR}/${PACKAGE_DIR}/conf" if file "${ROOTDIR}/${DIST_DIR}/${PACKAGE_DIR}/loongcollector" | grep x86-64; then ./scripts/download_ebpflib.sh "${ROOTDIR}/${DIST_DIR}/${PACKAGE_DIR}"; fi # Splitting debug info at build time with -gsplit-dwarf does not work with current gcc version diff --git a/scripts/gen_build_scripts.sh b/scripts/gen_build_scripts.sh index f1116b894b..9ad32874d0 100755 --- a/scripts/gen_build_scripts.sh +++ b/scripts/gen_build_scripts.sh @@ -122,7 +122,7 @@ function generateCopyScript() { fi echo 'mkdir -p $BINDIR/conf/instance_config/local/' >>$COPY_SCRIPT_FILE echo 'echo -e "{\n}" > $BINDIR/conf/instance_config/local/loongcollector_config.json' >>$COPY_SCRIPT_FILE - echo 'mkdir -p $BINDIR/conf/pipeline_config/local' >>$COPY_SCRIPT_FILE + echo 'mkdir -p $BINDIR/conf/continuous_pipeline_config/local' >>$COPY_SCRIPT_FILE echo 'docker rm -v "$id"' >>$COPY_SCRIPT_FILE } diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml index f5f31279dc..47545cfb41 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtail/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/continuous_pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" diff --git a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml index f5f31279dc..47545cfb41 100644 --- a/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml +++ b/test/benchmark/test_cases/performance_file_to_blackhole_ilogtailspl/docker-compose.yaml @@ -18,7 +18,7 @@ services: ilogtailC: image: aliyun/loongcollector:0.0.1 volumes: - - ./ilogtail.yaml:/loongcollector/conf/pipeline_config/local/ilogtail.yaml + - ./ilogtail.yaml:/loongcollector/conf/continuous_pipeline_config/local/ilogtail.yaml - .:/home/ilogtail healthcheck: test: "cat /loongcollector/log/loongcollector.LOG" diff --git a/test/engine/setup/dockercompose/compose.go b/test/engine/setup/dockercompose/compose.go index 4163354568..780df589e5 100644 --- a/test/engine/setup/dockercompose/compose.go +++ b/test/engine/setup/dockercompose/compose.go @@ -62,7 +62,7 @@ services: pid: host volumes: - %s:/loongcollector/conf/default_flusher.json - - %s:/loongcollector/conf/pipeline_config/local + - %s:/loongcollector/conf/continuous_pipeline_config/local - /:/logtail_host - /var/run/docker.sock:/var/run/docker.sock - /sys/:/sys/ From 0ff6f895edf32642e2705f2ea46d5e162be9e09b Mon Sep 17 00:00:00 2001 From: quzard <1191890118@qq.com> Date: Mon, 25 Nov 2024 11:30:15 +0800 Subject: [PATCH 09/11] SymLinkName, PidFileName, and AgentPrefix support logtail_mode. (#1904) --- core/app_config/AppConfig.cpp | 35 ++++++++++++++++++++++++++++++++--- core/app_config/AppConfig.h | 3 +++ core/common/version.h | 3 --- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/core/app_config/AppConfig.cpp b/core/app_config/AppConfig.cpp index dc9b3820c7..d492af6b7b 100644 --- a/core/app_config/AppConfig.cpp +++ b/core/app_config/AppConfig.cpp @@ -25,6 +25,7 @@ #include "common/FileSystemUtil.h" #include "common/JsonUtil.h" #include "common/LogtailCommonFlags.h" +#include "common/version.h" #include "config/InstanceConfigManager.h" #include "config/watcher/InstanceConfigWatcher.h" #include "file_server/ConfigManager.h" @@ -39,6 +40,10 @@ using namespace std; +#define ILOGTAIL_PREFIX "ilogtail_" +#define ILOGTAIL_PIDFILE_SUFFIX ".pid" +#define LOONGCOLLECTOR_PREFIX "loongcollector_" + DEFINE_FLAG_BOOL(logtail_mode, "logtail mode", false); DEFINE_FLAG_INT32(max_buffer_num, "max size", 40); DEFINE_FLAG_INT32(pub_max_buffer_num, "max size", 8); @@ -520,7 +525,7 @@ std::string GetAgentName() { return "ilogtail"; } else { return "loongcollector"; - } + } } std::string GetMonitorInfoFileName() { @@ -531,6 +536,30 @@ std::string GetMonitorInfoFileName() { } } +std::string GetSymLinkName() { + if (BOOL_FLAG(logtail_mode)) { + return GetProcessExecutionDir() + "ilogtail"; + } else { + return GetProcessExecutionDir() + "loongcollector"; + } +} + +std::string GetPidFileName() { + if (BOOL_FLAG(logtail_mode)) { + return GetProcessExecutionDir() + ILOGTAIL_PREFIX + ILOGTAIL_VERSION + ILOGTAIL_PIDFILE_SUFFIX; + } else { + return GetAgentRunDir() + "loongcollector.pid"; + } +} + +std::string GetAgentPrefix() { + if (BOOL_FLAG(logtail_mode)) { + return ILOGTAIL_PREFIX; + } else { + return LOONGCOLLECTOR_PREFIX; + } +} + AppConfig::AppConfig() { LOG_INFO(sLogger, ("AppConfig AppConfig", "success")); SetIlogtailConfigJson(""); @@ -1509,8 +1538,8 @@ void AppConfig::ReadFlagsFromMap(const std::unordered_map sIgnoreKeySet = {"data_server_list", "legacy_data_server_list"}; - const static unordered_set sForceKeySet = {"config_server_address_list", "config_server_list"}; + const static unordered_set sIgnoreKeySet = {"data_server_list", "data_servers"}; + const static unordered_set sForceKeySet = {"config_server_address_list", "config_servers"}; for (auto name : confJson.getMemberNames()) { auto jsonvalue = confJson[name]; string fullName; diff --git a/core/app_config/AppConfig.h b/core/app_config/AppConfig.h index 6a970c921f..25cdb0fdcb 100644 --- a/core/app_config/AppConfig.h +++ b/core/app_config/AppConfig.h @@ -61,6 +61,9 @@ std::string GetVersionTag(); std::string GetGoPluginCheckpoint(); std::string GetAgentName(); std::string GetMonitorInfoFileName(); +std::string GetSymLinkName(); +std::string GetPidFileName(); +std::string GetAgentPrefix(); template class DoubleBuffer { diff --git a/core/common/version.h b/core/common/version.h index 6256797df1..bd72e863c7 100644 --- a/core/common/version.h +++ b/core/common/version.h @@ -22,7 +22,4 @@ extern const char* const ILOGTAIL_UPDATE_SUFFIX; extern const char* const ILOGTAIL_GIT_HASH; extern const char* const ILOGTAIL_BUILD_DATE; -#define ILOGTAIL_PREFIX "ilogtail_" -#define ILOGTAIL_PIDFILE_SUFFIX ".pid" - #endif \ No newline at end of file From b11fd8f130988798f1ef24ab6dbe26f389b3e49f Mon Sep 17 00:00:00 2001 From: Takuka0311 <1914426213@qq.com> Date: Mon, 25 Nov 2024 11:37:45 +0800 Subject: [PATCH 10/11] init (#1905) --- core/pipeline/Pipeline.cpp | 2 +- core/pipeline/batch/Batcher.h | 16 +++++++++++----- .../pipeline/plugin/instance/FlusherInstance.cpp | 3 ++- core/pipeline/plugin/instance/FlusherInstance.h | 5 +++-- core/pipeline/plugin/interface/Flusher.h | 3 +++ core/unittest/pipeline/PipelineUnittest.cpp | 12 ++++++------ core/unittest/plugin/FlusherInstanceUnittest.cpp | 2 +- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 359385c6c5..e4161db2ee 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -167,7 +167,7 @@ bool Pipeline::Init(PipelineConfig&& config) { = PluginRegistry::GetInstance()->CreateFlusher(pluginType, GenNextPluginMeta(false)); if (flusher) { Json::Value optionalGoPipeline; - if (!flusher->Init(detail, mContext, optionalGoPipeline)) { + if (!flusher->Init(detail, mContext, i, optionalGoPipeline)) { return false; } mFlushers.emplace_back(std::move(flusher)); diff --git a/core/pipeline/batch/Batcher.h b/core/pipeline/batch/Batcher.h index 0d47087ffe..a263d82228 100644 --- a/core/pipeline/batch/Batcher.h +++ b/core/pipeline/batch/Batcher.h @@ -176,7 +176,7 @@ class Batcher { } if (mGroupQueue->IsEmpty()) { TimeoutFlushManager::GetInstance()->UpdateRecord(mFlusher->GetContext().GetConfigName(), - 0, + mFlusher->GetFlusherIndex(), 0, mGroupFlushStrategy->GetTimeoutSecs(), mFlusher); @@ -193,8 +193,11 @@ class Batcher { g.GetSourceBuffer(), g.GetExactlyOnceCheckpoint(), g.GetMetadata(EventGroupMetaKey::SOURCE_ID)); - TimeoutFlushManager::GetInstance()->UpdateRecord( - mFlusher->GetContext().GetConfigName(), 0, key, mEventFlushStrategy.GetTimeoutSecs(), mFlusher); + TimeoutFlushManager::GetInstance()->UpdateRecord(mFlusher->GetContext().GetConfigName(), + mFlusher->GetFlusherIndex(), + key, + mEventFlushStrategy.GetTimeoutSecs(), + mFlusher); mBufferedGroupsTotal->Add(1); mBufferedDataSizeByte->Add(item.DataSize()); } else if (i == 0) { @@ -243,8 +246,11 @@ class Batcher { mGroupQueue->Flush(res); } if (mGroupQueue->IsEmpty()) { - TimeoutFlushManager::GetInstance()->UpdateRecord( - mFlusher->GetContext().GetConfigName(), 0, 0, mGroupFlushStrategy->GetTimeoutSecs(), mFlusher); + TimeoutFlushManager::GetInstance()->UpdateRecord(mFlusher->GetContext().GetConfigName(), + mFlusher->GetFlusherIndex(), + 0, + mGroupFlushStrategy->GetTimeoutSecs(), + mFlusher); } iter->second.Flush(mGroupQueue.value()); mEventQueueMap.erase(iter); diff --git a/core/pipeline/plugin/instance/FlusherInstance.cpp b/core/pipeline/plugin/instance/FlusherInstance.cpp index 38cb9dd3b8..181e5c21e8 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.cpp +++ b/core/pipeline/plugin/instance/FlusherInstance.cpp @@ -20,9 +20,10 @@ using namespace std; namespace logtail { -bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, Json::Value& optionalGoPipeline) { +bool FlusherInstance::Init(const Json::Value& config, PipelineContext& context, size_t flusherIdx, Json::Value& optionalGoPipeline) { mPlugin->SetContext(context); mPlugin->SetPluginID(PluginID()); + mPlugin->SetFlusherIndex(flusherIdx); mPlugin->SetMetricsRecordRef(Name(), PluginID()); if (!mPlugin->Init(config, optionalGoPipeline)) { return false; diff --git a/core/pipeline/plugin/instance/FlusherInstance.h b/core/pipeline/plugin/instance/FlusherInstance.h index 69bbba3db2..68089f60b1 100644 --- a/core/pipeline/plugin/instance/FlusherInstance.h +++ b/core/pipeline/plugin/instance/FlusherInstance.h @@ -31,12 +31,13 @@ namespace logtail { class FlusherInstance : public PluginInstance { public: - FlusherInstance(Flusher* plugin, const PluginInstance::PluginMeta& pluginMeta) : PluginInstance(pluginMeta), mPlugin(plugin) {} + FlusherInstance(Flusher* plugin, const PluginInstance::PluginMeta& pluginMeta) + : PluginInstance(pluginMeta), mPlugin(plugin) {} const std::string& Name() const override { return mPlugin->Name(); }; const Flusher* GetPlugin() const { return mPlugin.get(); } - bool Init(const Json::Value& config, PipelineContext& context, Json::Value& optionalGoPipeline); + bool Init(const Json::Value& config, PipelineContext& context, size_t flusherIdx, Json::Value& optionalGoPipeline); bool Start() { return mPlugin->Start(); } bool Stop(bool isPipelineRemoving) { return mPlugin->Stop(isPipelineRemoving); } bool Send(PipelineEventGroup&& g); diff --git a/core/pipeline/plugin/interface/Flusher.h b/core/pipeline/plugin/interface/Flusher.h index 6bf3301477..232020df34 100644 --- a/core/pipeline/plugin/interface/Flusher.h +++ b/core/pipeline/plugin/interface/Flusher.h @@ -44,6 +44,8 @@ class Flusher : public Plugin { QueueKey GetQueueKey() const { return mQueueKey; } void SetPluginID(const std::string& pluginID) { mPluginID = pluginID; } + size_t GetFlusherIndex() { return mIndex; } + void SetFlusherIndex(size_t idx) { mIndex = idx; } const std::string& GetPluginID() const { return mPluginID; } protected: @@ -54,6 +56,7 @@ class Flusher : public Plugin { QueueKey mQueueKey; std::string mPluginID; + size_t mIndex = 0; #ifdef APSARA_UNIT_TEST_MAIN friend class FlusherInstanceUnittest; diff --git a/core/unittest/pipeline/PipelineUnittest.cpp b/core/unittest/pipeline/PipelineUnittest.cpp index 8134336fc5..0f9b273a37 100644 --- a/core/unittest/pipeline/PipelineUnittest.cpp +++ b/core/unittest/pipeline/PipelineUnittest.cpp @@ -2736,13 +2736,13 @@ void PipelineUnittest::TestSend() const { { auto flusher = PluginRegistry::GetInstance()->CreateFlusher(FlusherMock::sName, pipeline.GenNextPluginMeta(false)); - flusher->Init(Json::Value(), ctx, tmp); + flusher->Init(Json::Value(), ctx, 0, tmp); pipeline.mFlushers.emplace_back(std::move(flusher)); } { auto flusher = PluginRegistry::GetInstance()->CreateFlusher(FlusherMock::sName, pipeline.GenNextPluginMeta(false)); - flusher->Init(Json::Value(), ctx, tmp); + flusher->Init(Json::Value(), ctx, 0, tmp); pipeline.mFlushers.emplace_back(std::move(flusher)); } vector> configs; @@ -2788,13 +2788,13 @@ void PipelineUnittest::TestSend() const { { auto flusher = PluginRegistry::GetInstance()->CreateFlusher(FlusherMock::sName, pipeline.GenNextPluginMeta(false)); - flusher->Init(Json::Value(), ctx, tmp); + flusher->Init(Json::Value(), ctx, 0, tmp); pipeline.mFlushers.emplace_back(std::move(flusher)); } { auto flusher = PluginRegistry::GetInstance()->CreateFlusher(FlusherMock::sName, pipeline.GenNextPluginMeta(false)); - flusher->Init(Json::Value(), ctx, tmp); + flusher->Init(Json::Value(), ctx, 0, tmp); pipeline.mFlushers.emplace_back(std::move(flusher)); } @@ -2855,13 +2855,13 @@ void PipelineUnittest::TestFlushBatch() const { { auto flusher = PluginRegistry::GetInstance()->CreateFlusher(FlusherMock::sName, pipeline.GenNextPluginMeta(false)); - flusher->Init(Json::Value(), ctx, tmp); + flusher->Init(Json::Value(), ctx, 0, tmp); pipeline.mFlushers.emplace_back(std::move(flusher)); } { auto flusher = PluginRegistry::GetInstance()->CreateFlusher(FlusherMock::sName, pipeline.GenNextPluginMeta(false)); - flusher->Init(Json::Value(), ctx, tmp); + flusher->Init(Json::Value(), ctx, 0, tmp); pipeline.mFlushers.emplace_back(std::move(flusher)); } { diff --git a/core/unittest/plugin/FlusherInstanceUnittest.cpp b/core/unittest/plugin/FlusherInstanceUnittest.cpp index b6fcc33633..800529df70 100644 --- a/core/unittest/plugin/FlusherInstanceUnittest.cpp +++ b/core/unittest/plugin/FlusherInstanceUnittest.cpp @@ -46,7 +46,7 @@ void FlusherInstanceUnittest::TestInit() const { = make_unique(new FlusherMock(), PluginInstance::PluginMeta("0")); Json::Value config, opt; PipelineContext context; - APSARA_TEST_TRUE(flusher->Init(config, context, opt)); + APSARA_TEST_TRUE(flusher->Init(config, context, 0, opt)); APSARA_TEST_EQUAL(&context, &flusher->GetPlugin()->GetContext()); } From 53ff4842efcab0d24d39ba4b542bf15c575720b7 Mon Sep 17 00:00:00 2001 From: Tom Yu Date: Mon, 25 Nov 2024 11:58:37 +0800 Subject: [PATCH 11/11] Enable OTLP Trace flushing to SLS (#1906) * enable otlp trace to SlsPb * fix UT --- docs/cn/plugins/input/service-otlp.md | 2 +- .../opentelemetry/otlpDataToSLSProto.go | 3 +- .../opentelemetry/service_otlp_v1_test.go | 102 +++++++++++++++++- 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/docs/cn/plugins/input/service-otlp.md b/docs/cn/plugins/input/service-otlp.md index a47b7b6aad..8ff61e6d85 100644 --- a/docs/cn/plugins/input/service-otlp.md +++ b/docs/cn/plugins/input/service-otlp.md @@ -2,7 +2,7 @@ ## 简介 -`service_otlp` `input`插件实现了`ServiceInputV1`和`ServiceInputV2`接口，可以接受`Opentelemetry log/metric/trace protocol`的http/gRPC请求，并且转换输出SLSProto或PipelineGroupEvents。目前尚不支持otlp trace转换到SLSProto。 +`service_otlp` `input`插件实现了`ServiceInputV1`和`ServiceInputV2`接口，可以接受`Opentelemetry log/metric/trace protocol`的http/gRPC请求，并且转换输出SLSProto或PipelineGroupEvents。 ## 版本 diff --git a/pkg/protocol/decoder/opentelemetry/otlpDataToSLSProto.go b/pkg/protocol/decoder/opentelemetry/otlpDataToSLSProto.go index cde29c88ad..a5adc16357 100644 --- a/pkg/protocol/decoder/opentelemetry/otlpDataToSLSProto.go +++ b/pkg/protocol/decoder/opentelemetry/otlpDataToSLSProto.go @@ -427,5 +427,6 @@ func ConvertOtlpTraceRequestV1(otlpTraceReq ptraceotlp.ExportRequest) (logs []*p } func ConvertOtlpTraceV1(otlpTrace ptrace.Traces) (logs []*protocol.Log, err error) { - return logs, fmt.Errorf("does_not_support_otlptraces") + log, _ := ConvertTrace(otlpTrace) + return log, nil } diff --git a/plugins/input/opentelemetry/service_otlp_v1_test.go b/plugins/input/opentelemetry/service_otlp_v1_test.go index 0f05a6ad67..d08033201e 100644 --- a/plugins/input/opentelemetry/service_otlp_v1_test.go +++ b/plugins/input/opentelemetry/service_otlp_v1_test.go @@ -17,7 +17,9 @@ package opentelemetry import ( "fmt" "net/http" + "strconv" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -203,7 +205,55 @@ func TestOtlpGRPC_Trace_V1(t *testing.T) { for i := 0; i < queueSize; i++ { err = exportTraces(cc, GenerateTraces(i+1)) - assert.Error(t, err, "does_not_support_otlptraces") + assert.NoError(t, err) + } + + starttm := time.Date(2020, 2, 11, 20, 26, 12, 321, time.UTC) + endtm := time.Date(2020, 2, 11, 20, 26, 13, 789, time.UTC) + traceID := "0102030405060708090a0b0c0d0e0f10" + spanID := "1112131415161718" + count := 0 + pos := 0 + for count < queueSize { + count++ + for i := pos; i < pos+count; i++ { + log := collector.Logs[i] + assert.Equal(t, int64(log.Time), endtm.Unix()) + assert.Equal(t, "host", log.Contents[0].Key) + assert.Equal(t, "service", log.Contents[1].Key) + assert.Equal(t, "resource", log.Contents[2].Key) + assert.Equal(t, "{\"resource-attr\":\"resource-attr-val-1\"}", log.Contents[2].Value) + assert.Equal(t, "otlp.name", log.Contents[3].Key) + assert.Equal(t, "otlp.version", log.Contents[4].Key) + assert.Equal(t, "traceID", log.Contents[5].Key) + assert.Equal(t, "spanID", log.Contents[6].Key) + assert.Equal(t, "parentSpanID", log.Contents[7].Key) + assert.Equal(t, "kind", log.Contents[8].Key) + assert.Equal(t, "name", log.Contents[9].Key) + assert.Equal(t, "links", log.Contents[10].Key) + assert.Equal(t, "logs", log.Contents[11].Key) + if (i-pos)%2 == 0 { + assert.Equal(t, traceID, log.Contents[5].Value) + assert.Equal(t, spanID, log.Contents[6].Value) + assert.Equal(t, "operationA", log.Contents[9].Value) + assert.Equal(t, "[{\"attribute\":{\"span-event-attr\":\"span-event-attr-val\"},\"name\":\"event-with-attr\",\"time\":1581452773000000123},{\"attribute\":{},\"name\":\"event\",\"time\":1581452773000000123}]", log.Contents[11].Value) + } else { + assert.Equal(t, []byte{}, []byte(log.Contents[5].Value)) + assert.Equal(t, []byte{}, []byte(log.Contents[6].Value)) + assert.Equal(t, "operationB", log.Contents[9].Value) + assert.Equal(t, "[{\"attribute\":{\"span-link-attr\":\"span-link-attr-val\"},\"spanID\":\"\",\"traceID\":\"\"},{\"attribute\":{},\"spanID\":\"\",\"traceID\":\"\"}]", log.Contents[10].Value) + } + assert.Equal(t, "traceState", log.Contents[12].Key) + assert.Equal(t, "start", log.Contents[13].Key) + assert.Equal(t, strconv.FormatInt(starttm.UnixMicro(), 10), log.Contents[13].Value) + assert.Equal(t, "end", log.Contents[14].Key) + assert.Equal(t, strconv.FormatInt(endtm.UnixMicro(), 10), log.Contents[14].Value) + assert.Equal(t, "duration", log.Contents[15].Key) + assert.Equal(t, "attribute", log.Contents[16].Key) + assert.Equal(t, "statusCode", log.Contents[17].Key) + assert.Equal(t, "statusMessage", log.Contents[18].Key) + } + pos += count } } @@ -312,6 +362,54 @@ func TestOtlpHTTP_Trace_V1(t *testing.T) { for i := 0; i < queueSize; i++ { req := ptraceotlp.NewExportRequestFromTraces(GenerateTraces(i + 1)) err = httpExport(client, req, url, i%2 == 0) - assert.Error(t, err, "does_not_support_otlptraces") + assert.NoError(t, err) + } + + starttm := time.Date(2020, 2, 11, 20, 26, 12, 321, time.UTC) + endtm := time.Date(2020, 2, 11, 20, 26, 13, 789, time.UTC) + traceID := "0102030405060708090a0b0c0d0e0f10" + spanID := "1112131415161718" + count := 0 + pos := 0 + for count < queueSize { + count++ + for i := pos; i < pos+count; i++ { + log := collector.Logs[i] + assert.Equal(t, int64(log.Time), endtm.Unix()) + assert.Equal(t, "host", log.Contents[0].Key) + assert.Equal(t, "service", log.Contents[1].Key) + assert.Equal(t, "resource", log.Contents[2].Key) + assert.Equal(t, "{\"resource-attr\":\"resource-attr-val-1\"}", log.Contents[2].Value) + assert.Equal(t, "otlp.name", log.Contents[3].Key) + assert.Equal(t, "otlp.version", log.Contents[4].Key) + assert.Equal(t, "traceID", log.Contents[5].Key) + assert.Equal(t, "spanID", log.Contents[6].Key) + assert.Equal(t, "parentSpanID", log.Contents[7].Key) + assert.Equal(t, "kind", log.Contents[8].Key) + assert.Equal(t, "name", log.Contents[9].Key) + assert.Equal(t, "links", log.Contents[10].Key) + assert.Equal(t, "logs", log.Contents[11].Key) + if (i-pos)%2 == 0 { + assert.Equal(t, traceID, log.Contents[5].Value) + assert.Equal(t, spanID, log.Contents[6].Value) + assert.Equal(t, "operationA", log.Contents[9].Value) + assert.Equal(t, "[{\"attribute\":{\"span-event-attr\":\"span-event-attr-val\"},\"name\":\"event-with-attr\",\"time\":1581452773000000123},{\"attribute\":{},\"name\":\"event\",\"time\":1581452773000000123}]", log.Contents[11].Value) + } else { + assert.Equal(t, []byte{}, []byte(log.Contents[5].Value)) + assert.Equal(t, []byte{}, []byte(log.Contents[6].Value)) + assert.Equal(t, "operationB", log.Contents[9].Value) + assert.Equal(t, "[{\"attribute\":{\"span-link-attr\":\"span-link-attr-val\"},\"spanID\":\"\",\"traceID\":\"\"},{\"attribute\":{},\"spanID\":\"\",\"traceID\":\"\"}]", log.Contents[10].Value) + } + assert.Equal(t, "traceState", log.Contents[12].Key) + assert.Equal(t, "start", log.Contents[13].Key) + assert.Equal(t, strconv.FormatInt(starttm.UnixMicro(), 10), log.Contents[13].Value) + assert.Equal(t, "end", log.Contents[14].Key) + assert.Equal(t, strconv.FormatInt(endtm.UnixMicro(), 10), log.Contents[14].Value) + assert.Equal(t, "duration", log.Contents[15].Key) + assert.Equal(t, "attribute", log.Contents[16].Key) + assert.Equal(t, "statusCode", log.Contents[17].Key) + assert.Equal(t, "statusMessage", log.Contents[18].Key) + } + pos += count } }