From a0e25158f56e7858cbf68255e94fb46cd2f94a16 Mon Sep 17 00:00:00 2001 From: Tindy X <49061470+tindy2013@users.noreply.github.com> Date: Tue, 5 Dec 2023 18:37:14 +0800 Subject: [PATCH] Enhancements Add no-resolve to generated Clash rule-set if exists (#679). Do not set CURLOPT_USERAGENT if User-Agent header is provided. Print curl verbose logs with standard format. Change thread id in logs to thread name. Optimize generation of rule contents. --- src/generator/config/ruleconvert.cpp | 71 ++++++++++++++++------------ src/generator/template/templates.cpp | 34 +++++++++---- src/handler/webget.cpp | 58 +++++++++++++++++------ src/utils/logger.cpp | 23 ++++++--- src/utils/string.h | 1 + 5 files changed, 130 insertions(+), 57 deletions(-) diff --git a/src/generator/config/ruleconvert.cpp b/src/generator/config/ruleconvert.cpp index ae992d958..f75070a27 100644 --- a/src/generator/config/ruleconvert.cpp +++ b/src/generator/config/ruleconvert.cpp @@ -95,6 +95,32 @@ std::string convertRuleset(const std::string &content, int type) } } +static std::string transformRuleToCommon(string_view_array &temp, const std::string &input, const std::string &group, bool no_resolve_only = false) +{ + temp.clear(); + std::string strLine; + split(temp, input, ','); + if(temp.size() < 2) + { + strLine = temp[0]; + strLine += ","; + strLine += group; + } + else + { + strLine = temp[0]; + strLine += ","; + strLine += temp[1]; + strLine += ","; + strLine += group; + if(temp.size() > 2 && (!no_resolve_only || temp[2] == "no-resolve")) + { + strLine += ","; + strLine += temp[2]; + } + } + return strLine; +} void rulesetToClash(YAML::Node &base_rule, std::vector &ruleset_content_array, bool overwrite_original_rules, bool new_field_name) { @@ -108,6 +134,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector &ruleset_ if(!overwrite_original_rules && base_rule[field_name].IsDefined()) rules = base_rule[field_name]; + std::vector temp(4); for(RulesetContent &x : ruleset_content_array) { if(global.maxAllowedRules && total_rules > global.maxAllowedRules) @@ -124,10 +151,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector &ruleset_ strLine = retrieved_rules.substr(2); if(startsWith(strLine, "FINAL")) strLine.replace(0, 5, "MATCH"); - strLine += "," + rule_group; - if(count_least(strLine, ',', 3)) - strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); - allRules.emplace_back(std::move(strLine)); + strLine = transformRuleToCommon(temp, strLine, rule_group); + allRules.emplace_back(strLine); total_rules++; continue; } @@ -152,11 +177,8 @@ void rulesetToClash(YAML::Node &base_rule, std::vector &ruleset_ strLine.erase(strLine.find("//")); strLine = trimWhitespace(strLine); } - strLine += "," + rule_group; - if(count_least(strLine, ',', 3)) - strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); - allRules.emplace_back(std::move(strLine)); - //rules.push_back(strLine); + strLine = transformRuleToCommon(temp, strLine, rule_group); + allRules.emplace_back(strLine); } } @@ -183,6 +205,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector } base_rule.remove(field_name); + string_view_array temp(4); for(RulesetContent &x : ruleset_content_array) { if(global.maxAllowedRules && total_rules > global.maxAllowedRules) @@ -199,9 +222,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector strLine = retrieved_rules.substr(2); if(startsWith(strLine, "FINAL")) strLine.replace(0, 5, "MATCH"); - strLine += "," + rule_group; - if(count_least(strLine, ',', 3)) - strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); + strLine = transformRuleToCommon(temp, strLine, rule_group); output_content += " - " + strLine + "\n"; total_rules++; continue; @@ -227,9 +248,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector strLine.erase(strLine.find("//")); strLine = trimWhitespace(strLine); } - strLine += "," + rule_group; - if(count_least(strLine, ',', 3)) - strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); + strLine = transformRuleToCommon(temp, strLine, rule_group); output_content += " - " + strLine + "\n"; total_rules++; } @@ -277,6 +296,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_c const std::string rule_match_regex = "^(.*?,.*?)(,.*)(,.*)$"; + string_view_array temp(4); for(RulesetContent &x : ruleset_content_array) { if(global.maxAllowedRules && total_rules > global.maxAllowedRules) @@ -289,21 +309,17 @@ void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_c strLine = x.rule_content.get().substr(2); if(strLine == "MATCH") strLine = "FINAL"; - strLine += "," + rule_group; if(surge_ver == -1 || surge_ver == -2) { - if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve") - strLine = regReplace(strLine, rule_match_regex, "$1$3$2"); - else - strLine = regReplace(strLine, rule_match_regex, "$1$3"); + strLine = transformRuleToCommon(temp, strLine, rule_group, true); } else { - if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3)) - strLine = regReplace(strLine, rule_match_regex, "$1$3$2"); + if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT")) + strLine = transformRuleToCommon(temp, strLine, rule_group); } strLine = replaceAllDistinct(strLine, ",,", ","); - allRules.emplace_back(std::move(strLine)); + allRules.emplace_back(strLine); total_rules++; continue; } @@ -436,15 +452,12 @@ void rulesetToSurge(INIReader &base_rule, std::vector &ruleset_c { if(startsWith(strLine, "IP-CIDR6")) strLine.replace(0, 8, "IP6-CIDR"); - if(count_least(strLine, ',', 3) && regReplace(strLine, rule_match_regex, "$2") == ",no-resolve") - strLine = regReplace(strLine, rule_match_regex, "$1$3$2"); - else - strLine = regReplace(strLine, rule_match_regex, "$1$3"); + strLine = transformRuleToCommon(temp, strLine, rule_group, true); } else { - if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT") && count_least(strLine, ',', 3)) - strLine = regReplace(strLine, rule_match_regex, "$1$3$2"); + if(!startsWith(strLine, "AND") && !startsWith(strLine, "OR") && !startsWith(strLine, "NOT")) + strLine = transformRuleToCommon(temp, strLine, rule_group); } allRules.emplace_back(std::move(strLine)); total_rules++; diff --git a/src/generator/template/templates.cpp b/src/generator/template/templates.cpp index a6d1c6e54..f04594947 100644 --- a/src/generator/template/templates.cpp +++ b/src/generator/template/templates.cpp @@ -378,7 +378,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rulese } if(!script) rules.emplace_back("RULE-SET," + rule_name + "," + rule_group); - groups.emplace_back(std::move(rule_name)); + groups.emplace_back(rule_name); continue; } if(!remote_path_prefix.empty()) @@ -398,7 +398,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rulese { if(!script) rules.emplace_back("RULE-SET," + rule_name + "," + rule_group); - groups.emplace_back(std::move(rule_name)); + groups.emplace_back(rule_name); continue; } } @@ -419,6 +419,7 @@ int renderClashScript(YAML::Node &base_rule, std::vector &rulese strStrm.clear(); strStrm< &rulese } else { - strLine += "," + rule_group; - if(count_least(strLine, ',', 3)) - strLine = regReplace(strLine, "^(.*?,.*?)(,.*)(,.*)$", "$1$3$2"); - rules.emplace_back(std::move(strLine)); + vArray = split(strLine, ","); + if(vArray.size() < 2) + { + strLine = vArray[0] + "," + rule_group; + } + else + { + strLine = vArray[0] + "," + vArray[1] + "," + rule_group; + if(vArray.size() > 2) + strLine += "," + vArray[2]; + } + rules.emplace_back(strLine); } } else if(!has_domain[rule_name] && (startsWith(strLine, "DOMAIN,") || startsWith(strLine, "DOMAIN-SUFFIX,"))) has_domain[rule_name] = true; else if(!has_ipcidr[rule_name] && (startsWith(strLine, "IP-CIDR,") || startsWith(strLine, "IP-CIDR6,"))) + { has_ipcidr[rule_name] = true; + if(strLine.find(",no-resolve") != std::string::npos) + has_no_resolve = true; + } } if(has_domain[rule_name] && !script) rules.emplace_back("RULE-SET," + rule_name + "_domain," + rule_group); if(has_ipcidr[rule_name] && !script) - rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group); + { + if(has_no_resolve) + rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group + ",no-resolve"); + else + rules.emplace_back("RULE-SET," + rule_name + "_ipcidr," + rule_group); + } if(std::find(groups.begin(), groups.end(), rule_name) == groups.end()) - groups.emplace_back(std::move(rule_name)); + groups.emplace_back(rule_name); } } for(std::string &x : groups) diff --git a/src/handler/webget.cpp b/src/handler/webget.cpp index c24fe98ac..d0431514d 100644 --- a/src/handler/webget.cpp +++ b/src/handler/webget.cpp @@ -94,7 +94,7 @@ class RWLock RWLock cache_rw_lock; //std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"; -static std::string user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION; +static auto user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION; struct curl_progress_data { @@ -113,7 +113,7 @@ static inline void curl_init() static int writer(char *data, size_t size, size_t nmemb, std::string *writerData) { - if(writerData == NULL) + if(writerData == nullptr) return 0; writerData->append(data, size*nmemb); @@ -133,7 +133,7 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur { if(clientp) { - curl_progress_data *data = reinterpret_cast(clientp); + auto *data = reinterpret_cast(clientp); if(data->size_limit) { if(dlnow > data->size_limit) @@ -143,10 +143,37 @@ static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, cur return 0; } +static int logger(CURL *handle, curl_infotype type, char *data, size_t size, void *userptr) +{ + (void)handle; + (void)userptr; + std::string prefix; + switch(type) + { + case CURLINFO_TEXT: + prefix = "CURL_INFO"; + break; + case CURLINFO_HEADER_IN: + case CURLINFO_HEADER_OUT: + prefix = "CURL_HEADER"; + break; + case CURLINFO_DATA_IN: + case CURLINFO_DATA_OUT: + default: + return 0; + } + std::string content(data, size); + if(content.back() == '\n') + content.pop_back(); + writeLog(0, prefix + ": " + content, LOG_LEVEL_VERBOSE); + return 0; +} + static inline void curl_set_common_options(CURL *curl_handle, const char *url, curl_progress_data *data) { curl_easy_setopt(curl_handle, CURLOPT_URL, url); curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global.logLevel == LOG_LEVEL_VERBOSE ? 1L : 0L); + curl_easy_setopt(curl_handle, CURLOPT_DEBUGFUNCTION, logger); curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L); curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L); curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L); @@ -154,7 +181,6 @@ static inline void curl_set_common_options(CURL *curl_handle, const char *url, c curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 0L); curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, 15L); - curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str.data()); curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, ""); if(data) { @@ -170,18 +196,18 @@ static int curlGet(const FetchArgument &argument, FetchResult &result) { CURL *curl_handle; std::string *data = result.content, new_url = argument.url; - struct curl_slist *list = NULL; - defer(curl_slist_free_all(list);) + curl_slist *header_list = nullptr; + defer(curl_slist_free_all(header_list);) long retVal = 0; curl_init(); curl_handle = curl_easy_init(); - if(argument.proxy.size()) + if(!argument.proxy.empty()) { if(startsWith(argument.proxy, "cors:")) { - list = curl_slist_append(list, "X-Requested-With: subconverter " VERSION); + header_list = curl_slist_append(header_list, "X-Requested-With: subconverter " VERSION); new_url = argument.proxy.substr(5) + argument.url; } else @@ -190,16 +216,20 @@ static int curlGet(const FetchArgument &argument, FetchResult &result) curl_progress_data limit; limit.size_limit = global.maxAllowedDownloadSize; curl_set_common_options(curl_handle, new_url.data(), &limit); - list = curl_slist_append(list, "Content-Type: application/json;charset=utf-8"); + header_list = curl_slist_append(header_list, "Content-Type: application/json;charset=utf-8"); if(argument.request_headers) { for(auto &x : *argument.request_headers) - list = curl_slist_append(list, (x.first + ": " + x.second).data()); + { + header_list = curl_slist_append(header_list, (x.first + ": " + x.second).data()); + } + if(!argument.request_headers->contains("User-Agent")) + curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, user_agent_str); } - list = curl_slist_append(list, "SubConverter-Request: 1"); - list = curl_slist_append(list, "SubConverter-Version: " VERSION); - if(list) - curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, list); + header_list = curl_slist_append(header_list, "SubConverter-Request: 1"); + header_list = curl_slist_append(header_list, "SubConverter-Version: " VERSION); + if(header_list) + curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, header_list); if(result.content) { diff --git a/src/utils/logger.cpp b/src/utils/logger.cpp index a51ebe60c..4da73e98c 100644 --- a/src/utils/logger.cpp +++ b/src/utils/logger.cpp @@ -13,10 +13,10 @@ std::string getTime(int type) time_t lt; char tmpbuf[32], cMillis[7]; std::string format; - timeval tv; - gettimeofday(&tv, NULL); + timeval tv = {}; + gettimeofday(&tv, nullptr); snprintf(cMillis, 7, "%.6ld", (long)tv.tv_usec); - lt = time(NULL); + lt = time(nullptr); struct tm *local = localtime(<); switch(type) { @@ -27,19 +27,30 @@ std::string getTime(int type) format = "%Y/%m/%d %a %H:%M:%S." + std::string(cMillis); break; case 3: + default: format = "%Y-%m-%d %H:%M:%S"; break; } strftime(tmpbuf, 32, format.data(), local); - return std::string(tmpbuf); + return {tmpbuf}; } +static std::string get_thread_name() +{ + static std::atomic_int counter = 0; + thread_local static std::string name = "Thread-" + std::to_string(counter++); + return name; +} + +std::mutex log_mutex; + void writeLog(int type, const std::string &content, int level) { if(level > global.logLevel) return; + std::lock_guard lock(log_mutex); const char *levels[] = {"[FATL]", "[ERRO]", "[WARN]", "[INFO]", "[DEBG]", "[VERB]"}; - std::cerr< res { - abi::__cxa_demangle(name, NULL, NULL, &status), + abi::__cxa_demangle(name, nullptr, nullptr, &status), std::free }; return (status == 0) ? res.get() : name; diff --git a/src/utils/string.h b/src/utils/string.h index cf820bb68..c7cdb5ea3 100644 --- a/src/utils/string.h +++ b/src/utils/string.h @@ -10,6 +10,7 @@ using string = std::string; using string_size = std::string::size_type; using string_array = std::vector; +using string_view_array = std::vector; using string_map = std::map; using string_multimap = std::multimap; using string_pair_array = std::vector>;