diff --git a/Jenkinsfile b/Jenkinsfile index 42db027f328aac..737c68588c4d6d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -81,6 +81,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps) def extra = extra_env.collect { "export ${it}" }.join('\n'); def branch = env.BRANCH_NAME ?: 'master'; + def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename', label: 'Getting changes' lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') { docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') { @@ -91,9 +92,9 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps) device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh")) } steps.each { item -> - if (branch != "master" && item.size() == 3 && !hasPathChanged(item[2])) { + if (branch != "master" && item.size() == 3 && !hasPathChanged(gitDiff, item[2])) { println "Skipping ${item[0]}: no changes in ${item[2]}." - return; + return } else { device(device_ip, item[0], item[1]) } @@ -104,29 +105,13 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps) } } -@NonCPS -def hasPathChanged(List paths) { - changedFiles = [] - for (changeLogSet in currentBuild.changeSets) { - for (entry in changeLogSet.getItems()) { - for (file in entry.getAffectedFiles()) { - changedFiles.add(file.getPath()) - } - } - } - - env.CHANGED_FILES = changedFiles.join(" ") - if (currentBuild.number > 1) { - env.CHANGED_FILES += currentBuild.previousBuild.getBuildVariables().get("CHANGED_FILES") - } - +def hasPathChanged(String gitDiff, List paths) { for (path in paths) { - if (env.CHANGED_FILES.contains(path)) { - return true; + if (gitDiff.contains(path)) { + return true } } - - return false; + return false } def setupCredentials() { diff --git a/tools/cabana/streams/replaystream.cc b/tools/cabana/streams/replaystream.cc index 293b363532f29c..18b0317d4637f9 100644 --- a/tools/cabana/streams/replaystream.cc +++ b/tools/cabana/streams/replaystream.cc @@ -49,8 +49,8 @@ void ReplayStream::mergeSegments() { } bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) { - replay.reset(new Replay(route, {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"}, - {}, nullptr, replay_flags, data_dir, this)); + replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"}, + {}, nullptr, replay_flags, data_dir.toStdString(), this)); replay->setSegmentCacheLimit(settings.max_cached_minutes); replay->installEventFilter(event_filter, this); QObject::connect(replay.get(), &Replay::seeking, this, &AbstractStream::seeking); @@ -153,7 +153,7 @@ AbstractStream *OpenReplayWidget::open() { route = route.mid(idx + 1); } - bool is_valid_format = Route::parseRoute(route).str.size() > 0; + bool is_valid_format = Route::parseRoute(route.toStdString()).str.size() > 0; if (!is_valid_format) { QMessageBox::warning(nullptr, tr("Warning"), tr("Invalid route format: '%1'").arg(route)); } else { diff --git a/tools/cabana/streams/replaystream.h b/tools/cabana/streams/replaystream.h index 25d6ef4a72eda1..2a9f343645d255 100644 --- a/tools/cabana/streams/replaystream.h +++ b/tools/cabana/streams/replaystream.h @@ -20,7 +20,7 @@ class ReplayStream : public AbstractStream { bool eventFilter(const Event *event); void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); } bool liveStreaming() const override { return false; } - inline QString routeName() const override { return replay->route()->name(); } + inline QString routeName() const override { return QString::fromStdString(replay->route()->name()); } inline QString carFingerprint() const override { return replay->carFingerprint().c_str(); } double minSeconds() const override { return replay->minSeconds(); } double maxSeconds() const { return replay->maxSeconds(); } diff --git a/tools/replay/consoleui.cc b/tools/replay/consoleui.cc index 21ca0ad74b8468..5d2cf409d9c8ad 100644 --- a/tools/replay/consoleui.cc +++ b/tools/replay/consoleui.cc @@ -247,7 +247,7 @@ void ConsoleUI::updateProgressBar(uint64_t cur, uint64_t total, bool success) { void ConsoleUI::updateSummary() { const auto &route = replay->route(); - mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", qPrintable(route->name()), route->segments().size()); + mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", route->name().c_str(), route->segments().size()); mvwprintw(w[Win::Stats], 1, 0, "Car Fingerprint: %s", replay->carFingerprint().c_str()); wrefresh(w[Win::Stats]); } diff --git a/tools/replay/main.cc b/tools/replay/main.cc index a0c072438dfdc9..4e11ddb58333f6 100644 --- a/tools/replay/main.cc +++ b/tools/replay/main.cc @@ -1,83 +1,155 @@ +#include + #include -#include +#include +#include +#include +#include #include "common/prefix.h" #include "tools/replay/consoleui.h" #include "tools/replay/replay.h" +#include "tools/replay/util.h" -int main(int argc, char *argv[]) { -#ifdef __APPLE__ - // With all sockets opened, we might hit the default limit of 256 on macOS - util::set_file_descriptor_limit(1024); -#endif +const std::string helpText = +R"(Usage: replay [options] +Options: + -a, --allow Whitelist of services to send + -b, --block Blacklist of services to send + -c, --cache Cache segments in memory. Default is 5 + -s, --start Start from + -x, --playback Playback + --demo Use a demo route instead of providing your own + -d, --data_dir Local directory with routes + -p, --prefix Set OPENPILOT_PREFIX + --dcam Load driver camera + --ecam Load wide road camera + --no-loop Stop at the end of the route + --no-cache Turn off local cache + --qcam Load qcamera + --no-hw-decoder Disable HW video decoding + --no-vipc Do not output video + --all Output all messages including uiDebug, userFlag + -h, --help Show this help message +)"; - QCoreApplication app(argc, argv); +struct ReplayConfig { + std::string route; + std::vector allow; + std::vector block; + std::string data_dir; + std::string prefix; + uint32_t flags = REPLAY_FLAG_NONE; + int start_seconds = 0; + int cache_segments = -1; + float playback_speed = -1; +}; + +bool parseArgs(int argc, char *argv[], ReplayConfig &config) { + const struct option cli_options[] = { + {"allow", required_argument, nullptr, 'a'}, + {"block", required_argument, nullptr, 'b'}, + {"cache", required_argument, nullptr, 'c'}, + {"start", required_argument, nullptr, 's'}, + {"playback", required_argument, nullptr, 'x'}, + {"demo", no_argument, nullptr, 0}, + {"data_dir", required_argument, nullptr, 'd'}, + {"prefix", required_argument, nullptr, 'p'}, + {"dcam", no_argument, nullptr, 0}, + {"ecam", no_argument, nullptr, 0}, + {"no-loop", no_argument, nullptr, 0}, + {"no-cache", no_argument, nullptr, 0}, + {"qcam", no_argument, nullptr, 0}, + {"no-hw-decoder", no_argument, nullptr, 0}, + {"no-vipc", no_argument, nullptr, 0}, + {"all", no_argument, nullptr, 0}, + {"help", no_argument, nullptr, 'h'}, + {nullptr, 0, nullptr, 0}, // Terminating entry + }; - const std::tuple flags[] = { - {"dcam", REPLAY_FLAG_DCAM, "load driver camera"}, - {"ecam", REPLAY_FLAG_ECAM, "load wide road camera"}, - {"no-loop", REPLAY_FLAG_NO_LOOP, "stop at the end of the route"}, - {"no-cache", REPLAY_FLAG_NO_FILE_CACHE, "turn off local cache"}, - {"qcam", REPLAY_FLAG_QCAMERA, "load qcamera"}, - {"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER, "disable HW video decoding"}, - {"no-vipc", REPLAY_FLAG_NO_VIPC, "do not output video"}, - {"all", REPLAY_FLAG_ALL_SERVICES, "do output all messages including uiDebug, userFlag" - ". this may causes issues when used along with UI"} + const std::map flag_map = { + {"dcam", REPLAY_FLAG_DCAM}, + {"ecam", REPLAY_FLAG_ECAM}, + {"no-loop", REPLAY_FLAG_NO_LOOP}, + {"no-cache", REPLAY_FLAG_NO_FILE_CACHE}, + {"qcam", REPLAY_FLAG_QCAMERA}, + {"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER}, + {"no-vipc", REPLAY_FLAG_NO_VIPC}, + {"all", REPLAY_FLAG_ALL_SERVICES}, }; - QCommandLineParser parser; - parser.setApplicationDescription("Mock openpilot components by publishing logged messages."); - parser.addHelpOption(); - parser.addPositionalArgument("route", "the drive to replay. find your drives at connect.comma.ai"); - parser.addOption({{"a", "allow"}, "whitelist of services to send", "allow"}); - parser.addOption({{"b", "block"}, "blacklist of services to send", "block"}); - parser.addOption({{"c", "cache"}, "cache segments in memory. default is 5", "n"}); - parser.addOption({{"s", "start"}, "start from ", "seconds"}); - parser.addOption({"x", QString("playback . between %1 - %2") - .arg(ConsoleUI::speed_array.front()).arg(ConsoleUI::speed_array.back()), "speed"}); - parser.addOption({"demo", "use a demo route instead of providing your own"}); - parser.addOption({"data_dir", "local directory with routes", "data_dir"}); - parser.addOption({"prefix", "set OPENPILOT_PREFIX", "prefix"}); - for (auto &[name, _, desc] : flags) { - parser.addOption({name, desc}); + if (argc == 1) { + std::cout << helpText; + return false; } - parser.process(app); - const QStringList args = parser.positionalArguments(); - if (args.empty() && !parser.isSet("demo")) { - parser.showHelp(); + int opt, option_index = 0; + while ((opt = getopt_long(argc, argv, "a:b:c:s:x:d:p:h", cli_options, &option_index)) != -1) { + switch (opt) { + case 'a': config.allow = split(optarg, ','); break; + case 'b': config.block = split(optarg, ','); break; + case 'c': config.cache_segments = std::atoi(optarg); break; + case 's': config.start_seconds = std::atoi(optarg); break; + case 'x': config.playback_speed = std::atof(optarg); break; + case 'd': config.data_dir = optarg; break; + case 'p': config.prefix = optarg; break; + case 0: { + std::string name = cli_options[option_index].name; + if (name == "demo") { + config.route = DEMO_ROUTE; + } else { + config.flags |= flag_map.at(name); + } + break; + } + case 'h': std::cout << helpText; return false; + default: return false; + } } - const QString route = args.empty() ? DEMO_ROUTE : args.first(); - QStringList allow = parser.value("allow").isEmpty() ? QStringList{} : parser.value("allow").split(","); - QStringList block = parser.value("block").isEmpty() ? QStringList{} : parser.value("block").split(","); + // Check for a route name (first positional argument) + if (config.route.empty() && optind < argc) { + config.route = argv[optind]; + } - uint32_t replay_flags = REPLAY_FLAG_NONE; - for (const auto &[name, flag, _] : flags) { - if (parser.isSet(name)) { - replay_flags |= flag; - } + if (config.route.empty()) { + std::cerr << "No route provided. Use --help for usage information.\n"; + return false; + } + + return true; +} + +int main(int argc, char *argv[]) { +#ifdef __APPLE__ + // With all sockets opened, we might hit the default limit of 256 on macOS + util::set_file_descriptor_limit(1024); +#endif + + QCoreApplication app(argc, argv); + ReplayConfig config; + + if (!parseArgs(argc, argv, config)) { + return 1; } std::unique_ptr op_prefix; - auto prefix = parser.value("prefix"); - if (!prefix.isEmpty()) { - op_prefix.reset(new OpenpilotPrefix(prefix.toStdString())); + if (!config.prefix.empty()) { + op_prefix = std::make_unique(config.prefix); } - Replay *replay = new Replay(route, allow, block, nullptr, replay_flags, parser.value("data_dir"), &app); - if (!parser.value("c").isEmpty()) { - replay->setSegmentCacheLimit(parser.value("c").toInt()); + Replay *replay = new Replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir, &app); + if (config.cache_segments > 0) { + replay->setSegmentCacheLimit(config.cache_segments); } - if (!parser.value("x").isEmpty()) { - replay->setSpeed(std::clamp(parser.value("x").toFloat(), - ConsoleUI::speed_array.front(), ConsoleUI::speed_array.back())); + if (config.playback_speed > 0) { + replay->setSpeed(std::clamp(config.playback_speed, ConsoleUI::speed_array.front(), ConsoleUI::speed_array.back())); } if (!replay->load()) { - return 0; + return 1; } ConsoleUI console_ui(replay); - replay->start(parser.value("start").toInt()); + replay->start(config.start_seconds); return app.exec(); } diff --git a/tools/replay/replay.cc b/tools/replay/replay.cc index 34dd33bc1a477d..a0ae90c07928c1 100644 --- a/tools/replay/replay.cc +++ b/tools/replay/replay.cc @@ -11,36 +11,43 @@ static void interrupt_sleep_handler(int signal) {} -Replay::Replay(QString route, QStringList allow, QStringList block, SubMaster *sm_, - uint32_t flags, QString data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) { +Replay::Replay(const std::string &route, std::vector allow, std::vector block, SubMaster *sm_, + uint32_t flags, const std::string &data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) { // Register signal handler for SIGUSR1 std::signal(SIGUSR1, interrupt_sleep_handler); if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) { - block << "uiDebug" << "userFlag"; + block.insert(block.end(), {"uiDebug", "userFlag"}); } - auto event_struct = capnp::Schema::from().asStruct(); - sockets_.resize(event_struct.getUnionFields().size()); + + auto event_schema = capnp::Schema::from().asStruct(); + sockets_.resize(event_schema.getUnionFields().size()); + std::vector active_services; + for (const auto &[name, _] : services) { - if (!block.contains(name.c_str()) && (allow.empty() || allow.contains(name.c_str()))) { - uint16_t which = event_struct.getFieldByName(name).getProto().getDiscriminantValue(); + bool in_block = std::find(block.begin(), block.end(), name) != block.end(); + bool in_allow = std::find(allow.begin(), allow.end(), name) != allow.end(); + if (!in_block && (allow.empty() || in_allow)) { + uint16_t which = event_schema.getFieldByName(name).getProto().getDiscriminantValue(); sockets_[which] = name.c_str(); + active_services.push_back(name); } } - if (!allow.isEmpty()) { + + if (!allow.empty()) { for (int i = 0; i < sockets_.size(); ++i) { filters_.push_back(i == cereal::Event::Which::INIT_DATA || i == cereal::Event::Which::CAR_PARAMS || sockets_[i]); } } - std::vector s; - std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(s), - [](const char *name) { return name != nullptr; }); - qDebug() << "services " << s; - qDebug() << "loading route " << route; + rInfo("active services: %s", join(active_services, ',').c_str()); + rInfo("loading route %s", route.c_str()); if (sm == nullptr) { - pm = std::make_unique(s); + std::vector socket_names; + std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(socket_names), + [](const char *name) { return name != nullptr; }); + pm = std::make_unique(socket_names); } route_ = std::make_unique(route, data_dir); } @@ -68,23 +75,23 @@ void Replay::stop() { bool Replay::load() { if (!route_->load()) { - qCritical() << "failed to load route" << route_->name() - << "from" << (route_->dir().isEmpty() ? "server" : route_->dir()); + rError("failed to load route %s from %s", route_->name().c_str(), + route_->dir().empty() ? "server" : route_->dir().c_str()); return false; } for (auto &[n, f] : route_->segments()) { - bool has_log = !f.rlog.isEmpty() || !f.qlog.isEmpty(); - bool has_video = !f.road_cam.isEmpty() || !f.qcamera.isEmpty(); + bool has_log = !f.rlog.empty() || !f.qlog.empty(); + bool has_video = !f.road_cam.empty() || !f.qcamera.empty(); if (has_log && (has_video || hasFlag(REPLAY_FLAG_NO_VIPC))) { segments_.insert({n, nullptr}); } } if (segments_.empty()) { - qCritical() << "no valid segments in route" << route_->name(); + rInfo("no valid segments in route: %s", route_->name().c_str()); return false; } - rInfo("load route %s with %zu valid segments", qPrintable(route_->name()), segments_.size()); + rInfo("load route %s with %zu valid segments", route_->name().c_str(), segments_.size()); max_seconds_ = (segments_.rbegin()->first + 1) * 60; return true; } @@ -167,7 +174,7 @@ void Replay::buildTimeline() { const auto &route_segments = route_->segments(); for (auto it = route_segments.cbegin(); it != route_segments.cend() && !exit_; ++it) { std::shared_ptr log(new LogReader()); - if (!log->load(it->second.qlog.toStdString(), &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue; + if (!log->load(it->second.qlog, &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue; std::vector> timeline; for (const Event &e : log->events) { diff --git a/tools/replay/replay.h b/tools/replay/replay.h index ac829ffb71b752..8d2e07ba5b6033 100644 --- a/tools/replay/replay.h +++ b/tools/replay/replay.h @@ -15,7 +15,7 @@ #include "tools/replay/camera.h" #include "tools/replay/route.h" -const QString DEMO_ROUTE = "a2a0ccea32023010|2023-07-27--13-01-19"; +#define DEMO_ROUTE "a2a0ccea32023010|2023-07-27--13-01-19" // one segment uses about 100M of memory constexpr int MIN_SEGMENTS_CACHE = 5; @@ -49,8 +49,8 @@ class Replay : public QObject { Q_OBJECT public: - Replay(QString route, QStringList allow, QStringList block, SubMaster *sm = nullptr, - uint32_t flags = REPLAY_FLAG_NONE, QString data_dir = "", QObject *parent = 0); + Replay(const std::string &route, std::vector allow, std::vector block, SubMaster *sm = nullptr, + uint32_t flags = REPLAY_FLAG_NONE, const std::string &data_dir = "", QObject *parent = 0); ~Replay(); bool load(); RouteLoadError lastRouteError() const { return route_->lastError(); } diff --git a/tools/replay/route.cc b/tools/replay/route.cc index c1e65526cf7b5e..e298b88b79a821 100644 --- a/tools/replay/route.cc +++ b/tools/replay/route.cc @@ -4,35 +4,44 @@ #include #include #include -#include #include #include +#include +#include #include "selfdrive/ui/qt/api.h" #include "system/hardware/hw.h" #include "tools/replay/replay.h" #include "tools/replay/util.h" -Route::Route(const QString &route, const QString &data_dir) : data_dir_(data_dir) { +Route::Route(const std::string &route, const std::string &data_dir) : data_dir_(data_dir) { route_ = parseRoute(route); } -RouteIdentifier Route::parseRoute(const QString &str) { +RouteIdentifier Route::parseRoute(const std::string &str) { RouteIdentifier identifier = {}; - QRegularExpression rx(R"(^((?[a-z0-9]{16})[|_/])?(?.{20})((?--|/)(?((-?\d+(:(-?\d+)?)?)|(:-?\d+))))?$)"); - if (auto match = rx.match(str); match.hasMatch()) { - identifier.dongle_id = match.captured("dongle_id"); - identifier.timestamp = match.captured("timestamp"); + static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?$)"); + std::smatch match; + + if (std::regex_match(str, match, pattern)) { + identifier.dongle_id = match[2].str(); + identifier.timestamp = match[3].str(); identifier.str = identifier.dongle_id + "|" + identifier.timestamp; - auto range_str = match.captured("range"); - if (auto separator = match.captured("separator"); separator == "/" && !range_str.isEmpty()) { - auto range = range_str.split(":"); - identifier.begin_segment = identifier.end_segment = range[0].toInt(); - if (range.size() == 2) { - identifier.end_segment = range[1].isEmpty() ? -1 : range[1].toInt(); + + const auto separator = match[5].str(); + const auto range_str = match[6].str(); + if (!range_str.empty()) { + if (separator == "/") { + int pos = range_str.find(':'); + int begin_seg = std::stoi(range_str.substr(0, pos)); + identifier.begin_segment = identifier.end_segment = begin_seg; + if (pos != std::string::npos) { + auto end_seg_str = range_str.substr(pos + 1); + identifier.end_segment = end_seg_str.empty() ? -1 : std::stoi(end_seg_str); + } + } else if (separator == "--") { + identifier.begin_segment = std::atoi(range_str.c_str()); } - } else if (separator == "--") { - identifier.begin_segment = range_str.toInt(); } } return identifier; @@ -40,12 +49,12 @@ RouteIdentifier Route::parseRoute(const QString &str) { bool Route::load() { err_ = RouteLoadError::None; - if (route_.str.isEmpty() || (data_dir_.isEmpty() && route_.dongle_id.isEmpty())) { + if (route_.str.empty() || (data_dir_.empty() && route_.dongle_id.empty())) { rInfo("invalid route format"); return false; } - date_time_ = QDateTime::fromString(route_.timestamp, "yyyy-MM-dd--HH-mm-ss"); - bool ret = data_dir_.isEmpty() ? loadFromServer() : loadFromLocal(); + date_time_ = QDateTime::fromString(route_.timestamp.c_str(), "yyyy-MM-dd--HH-mm-ss"); + bool ret = data_dir_.empty() ? loadFromServer() : loadFromLocal(); if (ret) { if (route_.begin_segment == -1) route_.begin_segment = segments_.rbegin()->first; if (route_.end_segment == -1) route_.end_segment = segments_.rbegin()->first; @@ -69,7 +78,7 @@ bool Route::loadFromServer(int retries) { result = json; loop.exit((int)err); }); - http.sendRequest(CommaApi::BASE_URL + "/v1/route/" + route_.str + "/files"); + http.sendRequest(CommaApi::BASE_URL + "/v1/route/" + QString::fromStdString(route_.str) + "/files"); auto err = (QNetworkReply::NetworkError)loop.exec(); if (err == QNetworkReply::NoError) { return loadFromJson(result); @@ -96,7 +105,7 @@ bool Route::loadFromJson(const QString &json) { for (const auto &url : value.toArray()) { QString url_str = url.toString(); if (rx.indexIn(url_str) != -1) { - addFileToSegment(rx.cap(1).toInt(), url_str); + addFileToSegment(rx.cap(1).toInt(), url_str.toStdString()); } } } @@ -104,23 +113,26 @@ bool Route::loadFromJson(const QString &json) { } bool Route::loadFromLocal() { - QDirIterator it(data_dir_, {QString("%1--*").arg(route_.timestamp)}, QDir::Dirs | QDir::NoDotAndDotDot); - while (it.hasNext()) { - QString segment = it.next(); - const int seg_num = segment.mid(segment.lastIndexOf("--") + 2).toInt(); - QDir segment_dir(segment); - for (const auto &f : segment_dir.entryList(QDir::Files)) { - addFileToSegment(seg_num, segment_dir.absoluteFilePath(f)); + std::string pattern = route_.timestamp + "--"; + for (const auto &entry : std::filesystem::directory_iterator(data_dir_)) { + if (entry.is_directory() && entry.path().filename().string().find(pattern) == 0) { + std::string segment = entry.path().string(); + int seg_num = std::atoi(segment.substr(segment.rfind("--") + 2).c_str()); + + for (const auto &file : std::filesystem::directory_iterator(segment)) { + if (file.is_regular_file()) { + addFileToSegment(seg_num, file.path().string()); + } + } } } return !segments_.empty(); } -void Route::addFileToSegment(int n, const QString &file) { - QString name = QUrl(file).fileName(); - - const int pos = name.lastIndexOf("--"); - name = pos != -1 ? name.mid(pos + 2) : name; +void Route::addFileToSegment(int n, const std::string &file) { + std::string name = extractFileName(file); + auto pos = name.find_last_of("--"); + name = pos != std::string::npos ? name.substr(pos + 2) : name; if (name == "rlog.bz2" || name == "rlog.zst" || name == "rlog") { segments_[n].rlog = file; @@ -143,15 +155,15 @@ Segment::Segment(int n, const SegmentFile &files, uint32_t flags, const std::vec : seg_num(n), flags(flags), filters_(filters) { // [RoadCam, DriverCam, WideRoadCam, log]. fallback to qcamera/qlog const std::array file_list = { - (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.isEmpty() ? files.qcamera : files.road_cam, + (flags & REPLAY_FLAG_QCAMERA) || files.road_cam.empty() ? files.qcamera : files.road_cam, flags & REPLAY_FLAG_DCAM ? files.driver_cam : "", flags & REPLAY_FLAG_ECAM ? files.wide_road_cam : "", - files.rlog.isEmpty() ? files.qlog : files.rlog, + files.rlog.empty() ? files.qlog : files.rlog, }; for (int i = 0; i < file_list.size(); ++i) { - if (!file_list[i].isEmpty() && (!(flags & REPLAY_FLAG_NO_VIPC) || i >= MAX_CAMERAS)) { + if (!file_list[i].empty() && (!(flags & REPLAY_FLAG_NO_VIPC) || i >= MAX_CAMERAS)) { ++loading_; - synchronizer_.addFuture(QtConcurrent::run(this, &Segment::loadFile, i, file_list[i].toStdString())); + synchronizer_.addFuture(QtConcurrent::run(this, &Segment::loadFile, i, file_list[i])); } } } diff --git a/tools/replay/route.h b/tools/replay/route.h index acc73d509a1f6d..9c1cc37e2a9ac5 100644 --- a/tools/replay/route.h +++ b/tools/replay/route.h @@ -21,42 +21,42 @@ enum class RouteLoadError { }; struct RouteIdentifier { - QString dongle_id; - QString timestamp; + std::string dongle_id; + std::string timestamp; int begin_segment = 0; int end_segment = -1; - QString str; + std::string str; }; struct SegmentFile { - QString rlog; - QString qlog; - QString road_cam; - QString driver_cam; - QString wide_road_cam; - QString qcamera; + std::string rlog; + std::string qlog; + std::string road_cam; + std::string driver_cam; + std::string wide_road_cam; + std::string qcamera; }; class Route { public: - Route(const QString &route, const QString &data_dir = {}); + Route(const std::string &route, const std::string &data_dir = {}); bool load(); RouteLoadError lastError() const { return err_; } - inline const QString &name() const { return route_.str; } + inline const std::string &name() const { return route_.str; } inline const QDateTime datetime() const { return date_time_; } - inline const QString &dir() const { return data_dir_; } + inline const std::string &dir() const { return data_dir_; } inline const RouteIdentifier &identifier() const { return route_; } inline const std::map &segments() const { return segments_; } inline const SegmentFile &at(int n) { return segments_.at(n); } - static RouteIdentifier parseRoute(const QString &str); + static RouteIdentifier parseRoute(const std::string &str); protected: bool loadFromLocal(); bool loadFromServer(int retries = 3); bool loadFromJson(const QString &json); - void addFileToSegment(int seg_num, const QString &file); + void addFileToSegment(int seg_num, const std::string &file); RouteIdentifier route_ = {}; - QString data_dir_; + std::string data_dir_; std::map segments_; QDateTime date_time_; RouteLoadError err_ = RouteLoadError::None; diff --git a/tools/replay/tests/test_replay.cc b/tools/replay/tests/test_replay.cc index 3e5bfdf984e309..2fe468e726d23c 100644 --- a/tools/replay/tests/test_replay.cc +++ b/tools/replay/tests/test_replay.cc @@ -122,12 +122,12 @@ std::string download_demo_route() { assert(remote_route.load()); // Create a local route from remote for testing - const std::string route_name = DEMO_ROUTE.mid(17).toStdString(); + const std::string route_name = std::string(DEMO_ROUTE).substr(17); for (int i = 0; i < 2; ++i) { std::string log_path = util::string_format("%s/%s--%d/", data_dir.c_str(), route_name.c_str(), i); util::create_directories(log_path, 0755); - REQUIRE(download_to_file(remote_route.at(i).rlog.toStdString(), log_path + "rlog.bz2")); - REQUIRE(download_to_file(remote_route.at(i).qcamera.toStdString(), log_path + "qcamera.ts")); + REQUIRE(download_to_file(remote_route.at(i).rlog, log_path + "rlog.bz2")); + REQUIRE(download_to_file(remote_route.at(i).qcamera, log_path + "qcamera.ts")); } } @@ -139,7 +139,7 @@ TEST_CASE("Local route") { std::string data_dir = download_demo_route(); auto flags = GENERATE(0, REPLAY_FLAG_QCAMERA); - Route route(DEMO_ROUTE, QString::fromStdString(data_dir)); + Route route(DEMO_ROUTE, data_dir); REQUIRE(route.load()); REQUIRE(route.segments().size() == 2); for (int i = 0; i < TEST_REPLAY_SEGMENTS; ++i) { diff --git a/tools/replay/util.cc b/tools/replay/util.cc index 0ae0c05f9b8dab..91f6af4f845575 100644 --- a/tools/replay/util.cc +++ b/tools/replay/util.cc @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -390,6 +391,35 @@ std::string sha256(const std::string &str) { return util::hexdump(hash, SHA256_DIGEST_LENGTH); } +std::vector split(std::string_view source, char delimiter) { + std::vector fields; + size_t last = 0; + for (size_t i = 0; i < source.length(); ++i) { + if (source[i] == delimiter) { + fields.emplace_back(source.substr(last, i - last)); + last = i + 1; + } + } + fields.emplace_back(source.substr(last)); + return fields; +} + +std::string join(const std::vector &elements, char separator) { + std::ostringstream oss; + for (size_t i = 0; i < elements.size(); ++i) { + if (i != 0) oss << separator; + oss << elements[i]; + } + return oss.str(); +} + +std::string extractFileName(const std::string &file) { + size_t queryPos = file.find_first_of("?"); + std::string path = (queryPos != std::string::npos) ? file.substr(0, queryPos) : file; + size_t lastSlash = path.find_last_of("/\\"); + return (lastSlash != std::string::npos) ? path.substr(lastSlash + 1) : path; +} + // MonotonicBuffer void *MonotonicBuffer::allocate(size_t bytes, size_t alignment) { diff --git a/tools/replay/util.h b/tools/replay/util.h index d6c26a34716ab1..317b964181a6a5 100644 --- a/tools/replay/util.h +++ b/tools/replay/util.h @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "cereal/messaging/messaging.h" enum CameraType { @@ -57,3 +59,6 @@ typedef std::function Download void installDownloadProgressHandler(DownloadProgressHandler); bool httpDownload(const std::string &url, const std::string &file, size_t chunk_size = 0, std::atomic *abort = nullptr); std::string formattedDataSize(size_t size); +std::vector split(std::string_view source, char delimiter); +std::string join(const std::vector &elements, char separator); +std::string extractFileName(const std::string& file);