Skip to content

Commit

Permalink
Merge branch 'commaai:master' into PA-dev2
Browse files Browse the repository at this point in the history
  • Loading branch information
Edison-CBS authored Oct 20, 2024
2 parents f57909b + 30853a2 commit d7051c4
Show file tree
Hide file tree
Showing 12 changed files with 272 additions and 161 deletions.
29 changes: 7 additions & 22 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)

def extra = extra_env.collect { "export ${it}" }.join('\n');
def branch = env.BRANCH_NAME ?: 'master';
def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename', label: 'Getting changes'

lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
Expand All @@ -91,9 +92,9 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
}
steps.each { item ->
if (branch != "master" && item.size() == 3 && !hasPathChanged(item[2])) {
if (branch != "master" && item.size() == 3 && !hasPathChanged(gitDiff, item[2])) {
println "Skipping ${item[0]}: no changes in ${item[2]}."
return;
return
} else {
device(device_ip, item[0], item[1])
}
Expand All @@ -104,29 +105,13 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
}
}

@NonCPS
def hasPathChanged(List<String> paths) {
changedFiles = []
for (changeLogSet in currentBuild.changeSets) {
for (entry in changeLogSet.getItems()) {
for (file in entry.getAffectedFiles()) {
changedFiles.add(file.getPath())
}
}
}

env.CHANGED_FILES = changedFiles.join(" ")
if (currentBuild.number > 1) {
env.CHANGED_FILES += currentBuild.previousBuild.getBuildVariables().get("CHANGED_FILES")
}

def hasPathChanged(String gitDiff, List<String> paths) {
for (path in paths) {
if (env.CHANGED_FILES.contains(path)) {
return true;
if (gitDiff.contains(path)) {
return true
}
}

return false;
return false
}

def setupCredentials() {
Expand Down
6 changes: 3 additions & 3 deletions tools/cabana/streams/replaystream.cc
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ void ReplayStream::mergeSegments() {
}

bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags) {
replay.reset(new Replay(route, {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"},
{}, nullptr, replay_flags, data_dir, this));
replay.reset(new Replay(route.toStdString(), {"can", "roadEncodeIdx", "driverEncodeIdx", "wideRoadEncodeIdx", "carParams"},
{}, nullptr, replay_flags, data_dir.toStdString(), this));
replay->setSegmentCacheLimit(settings.max_cached_minutes);
replay->installEventFilter(event_filter, this);
QObject::connect(replay.get(), &Replay::seeking, this, &AbstractStream::seeking);
Expand Down Expand Up @@ -153,7 +153,7 @@ AbstractStream *OpenReplayWidget::open() {
route = route.mid(idx + 1);
}

bool is_valid_format = Route::parseRoute(route).str.size() > 0;
bool is_valid_format = Route::parseRoute(route.toStdString()).str.size() > 0;
if (!is_valid_format) {
QMessageBox::warning(nullptr, tr("Warning"), tr("Invalid route format: '%1'").arg(route));
} else {
Expand Down
2 changes: 1 addition & 1 deletion tools/cabana/streams/replaystream.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class ReplayStream : public AbstractStream {
bool eventFilter(const Event *event);
void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); }
bool liveStreaming() const override { return false; }
inline QString routeName() const override { return replay->route()->name(); }
inline QString routeName() const override { return QString::fromStdString(replay->route()->name()); }
inline QString carFingerprint() const override { return replay->carFingerprint().c_str(); }
double minSeconds() const override { return replay->minSeconds(); }
double maxSeconds() const { return replay->maxSeconds(); }
Expand Down
2 changes: 1 addition & 1 deletion tools/replay/consoleui.cc
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ void ConsoleUI::updateProgressBar(uint64_t cur, uint64_t total, bool success) {

void ConsoleUI::updateSummary() {
const auto &route = replay->route();
mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", qPrintable(route->name()), route->segments().size());
mvwprintw(w[Win::Stats], 0, 0, "Route: %s, %lu segments", route->name().c_str(), route->segments().size());
mvwprintw(w[Win::Stats], 1, 0, "Car Fingerprint: %s", replay->carFingerprint().c_str());
wrefresh(w[Win::Stats]);
}
Expand Down
182 changes: 127 additions & 55 deletions tools/replay/main.cc
Original file line number Diff line number Diff line change
@@ -1,83 +1,155 @@
#include <getopt.h>

#include <QApplication>
#include <QCommandLineParser>
#include <iostream>
#include <map>
#include <string>
#include <vector>

#include "common/prefix.h"
#include "tools/replay/consoleui.h"
#include "tools/replay/replay.h"
#include "tools/replay/util.h"

int main(int argc, char *argv[]) {
#ifdef __APPLE__
// With all sockets opened, we might hit the default limit of 256 on macOS
util::set_file_descriptor_limit(1024);
#endif
const std::string helpText =
R"(Usage: replay [options]
Options:
-a, --allow Whitelist of services to send
-b, --block Blacklist of services to send
-c, --cache Cache <n> segments in memory. Default is 5
-s, --start Start from <seconds>
-x, --playback Playback <speed>
--demo Use a demo route instead of providing your own
-d, --data_dir Local directory with routes
-p, --prefix Set OPENPILOT_PREFIX
--dcam Load driver camera
--ecam Load wide road camera
--no-loop Stop at the end of the route
--no-cache Turn off local cache
--qcam Load qcamera
--no-hw-decoder Disable HW video decoding
--no-vipc Do not output video
--all Output all messages including uiDebug, userFlag
-h, --help Show this help message
)";

QCoreApplication app(argc, argv);
struct ReplayConfig {
std::string route;
std::vector<std::string> allow;
std::vector<std::string> block;
std::string data_dir;
std::string prefix;
uint32_t flags = REPLAY_FLAG_NONE;
int start_seconds = 0;
int cache_segments = -1;
float playback_speed = -1;
};

bool parseArgs(int argc, char *argv[], ReplayConfig &config) {
const struct option cli_options[] = {
{"allow", required_argument, nullptr, 'a'},
{"block", required_argument, nullptr, 'b'},
{"cache", required_argument, nullptr, 'c'},
{"start", required_argument, nullptr, 's'},
{"playback", required_argument, nullptr, 'x'},
{"demo", no_argument, nullptr, 0},
{"data_dir", required_argument, nullptr, 'd'},
{"prefix", required_argument, nullptr, 'p'},
{"dcam", no_argument, nullptr, 0},
{"ecam", no_argument, nullptr, 0},
{"no-loop", no_argument, nullptr, 0},
{"no-cache", no_argument, nullptr, 0},
{"qcam", no_argument, nullptr, 0},
{"no-hw-decoder", no_argument, nullptr, 0},
{"no-vipc", no_argument, nullptr, 0},
{"all", no_argument, nullptr, 0},
{"help", no_argument, nullptr, 'h'},
{nullptr, 0, nullptr, 0}, // Terminating entry
};

const std::tuple<QString, REPLAY_FLAGS, QString> flags[] = {
{"dcam", REPLAY_FLAG_DCAM, "load driver camera"},
{"ecam", REPLAY_FLAG_ECAM, "load wide road camera"},
{"no-loop", REPLAY_FLAG_NO_LOOP, "stop at the end of the route"},
{"no-cache", REPLAY_FLAG_NO_FILE_CACHE, "turn off local cache"},
{"qcam", REPLAY_FLAG_QCAMERA, "load qcamera"},
{"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER, "disable HW video decoding"},
{"no-vipc", REPLAY_FLAG_NO_VIPC, "do not output video"},
{"all", REPLAY_FLAG_ALL_SERVICES, "do output all messages including uiDebug, userFlag"
". this may causes issues when used along with UI"}
const std::map<std::string, REPLAY_FLAGS> flag_map = {
{"dcam", REPLAY_FLAG_DCAM},
{"ecam", REPLAY_FLAG_ECAM},
{"no-loop", REPLAY_FLAG_NO_LOOP},
{"no-cache", REPLAY_FLAG_NO_FILE_CACHE},
{"qcam", REPLAY_FLAG_QCAMERA},
{"no-hw-decoder", REPLAY_FLAG_NO_HW_DECODER},
{"no-vipc", REPLAY_FLAG_NO_VIPC},
{"all", REPLAY_FLAG_ALL_SERVICES},
};

QCommandLineParser parser;
parser.setApplicationDescription("Mock openpilot components by publishing logged messages.");
parser.addHelpOption();
parser.addPositionalArgument("route", "the drive to replay. find your drives at connect.comma.ai");
parser.addOption({{"a", "allow"}, "whitelist of services to send", "allow"});
parser.addOption({{"b", "block"}, "blacklist of services to send", "block"});
parser.addOption({{"c", "cache"}, "cache <n> segments in memory. default is 5", "n"});
parser.addOption({{"s", "start"}, "start from <seconds>", "seconds"});
parser.addOption({"x", QString("playback <speed>. between %1 - %2")
.arg(ConsoleUI::speed_array.front()).arg(ConsoleUI::speed_array.back()), "speed"});
parser.addOption({"demo", "use a demo route instead of providing your own"});
parser.addOption({"data_dir", "local directory with routes", "data_dir"});
parser.addOption({"prefix", "set OPENPILOT_PREFIX", "prefix"});
for (auto &[name, _, desc] : flags) {
parser.addOption({name, desc});
if (argc == 1) {
std::cout << helpText;
return false;
}

parser.process(app);
const QStringList args = parser.positionalArguments();
if (args.empty() && !parser.isSet("demo")) {
parser.showHelp();
int opt, option_index = 0;
while ((opt = getopt_long(argc, argv, "a:b:c:s:x:d:p:h", cli_options, &option_index)) != -1) {
switch (opt) {
case 'a': config.allow = split(optarg, ','); break;
case 'b': config.block = split(optarg, ','); break;
case 'c': config.cache_segments = std::atoi(optarg); break;
case 's': config.start_seconds = std::atoi(optarg); break;
case 'x': config.playback_speed = std::atof(optarg); break;
case 'd': config.data_dir = optarg; break;
case 'p': config.prefix = optarg; break;
case 0: {
std::string name = cli_options[option_index].name;
if (name == "demo") {
config.route = DEMO_ROUTE;
} else {
config.flags |= flag_map.at(name);
}
break;
}
case 'h': std::cout << helpText; return false;
default: return false;
}
}

const QString route = args.empty() ? DEMO_ROUTE : args.first();
QStringList allow = parser.value("allow").isEmpty() ? QStringList{} : parser.value("allow").split(",");
QStringList block = parser.value("block").isEmpty() ? QStringList{} : parser.value("block").split(",");
// Check for a route name (first positional argument)
if (config.route.empty() && optind < argc) {
config.route = argv[optind];
}

uint32_t replay_flags = REPLAY_FLAG_NONE;
for (const auto &[name, flag, _] : flags) {
if (parser.isSet(name)) {
replay_flags |= flag;
}
if (config.route.empty()) {
std::cerr << "No route provided. Use --help for usage information.\n";
return false;
}

return true;
}

int main(int argc, char *argv[]) {
#ifdef __APPLE__
// With all sockets opened, we might hit the default limit of 256 on macOS
util::set_file_descriptor_limit(1024);
#endif

QCoreApplication app(argc, argv);
ReplayConfig config;

if (!parseArgs(argc, argv, config)) {
return 1;
}

std::unique_ptr<OpenpilotPrefix> op_prefix;
auto prefix = parser.value("prefix");
if (!prefix.isEmpty()) {
op_prefix.reset(new OpenpilotPrefix(prefix.toStdString()));
if (!config.prefix.empty()) {
op_prefix = std::make_unique<OpenpilotPrefix>(config.prefix);
}

Replay *replay = new Replay(route, allow, block, nullptr, replay_flags, parser.value("data_dir"), &app);
if (!parser.value("c").isEmpty()) {
replay->setSegmentCacheLimit(parser.value("c").toInt());
Replay *replay = new Replay(config.route, config.allow, config.block, nullptr, config.flags, config.data_dir, &app);
if (config.cache_segments > 0) {
replay->setSegmentCacheLimit(config.cache_segments);
}
if (!parser.value("x").isEmpty()) {
replay->setSpeed(std::clamp(parser.value("x").toFloat(),
ConsoleUI::speed_array.front(), ConsoleUI::speed_array.back()));
if (config.playback_speed > 0) {
replay->setSpeed(std::clamp(config.playback_speed, ConsoleUI::speed_array.front(), ConsoleUI::speed_array.back()));
}
if (!replay->load()) {
return 0;
return 1;
}

ConsoleUI console_ui(replay);
replay->start(parser.value("start").toInt());
replay->start(config.start_seconds);
return app.exec();
}
49 changes: 28 additions & 21 deletions tools/replay/replay.cc
Original file line number Diff line number Diff line change
Expand Up @@ -11,36 +11,43 @@

static void interrupt_sleep_handler(int signal) {}

Replay::Replay(QString route, QStringList allow, QStringList block, SubMaster *sm_,
uint32_t flags, QString data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) {
Replay::Replay(const std::string &route, std::vector<std::string> allow, std::vector<std::string> block, SubMaster *sm_,
uint32_t flags, const std::string &data_dir, QObject *parent) : sm(sm_), flags_(flags), QObject(parent) {
// Register signal handler for SIGUSR1
std::signal(SIGUSR1, interrupt_sleep_handler);

if (!(flags_ & REPLAY_FLAG_ALL_SERVICES)) {
block << "uiDebug" << "userFlag";
block.insert(block.end(), {"uiDebug", "userFlag"});
}
auto event_struct = capnp::Schema::from<cereal::Event>().asStruct();
sockets_.resize(event_struct.getUnionFields().size());

auto event_schema = capnp::Schema::from<cereal::Event>().asStruct();
sockets_.resize(event_schema.getUnionFields().size());
std::vector<std::string> active_services;

for (const auto &[name, _] : services) {
if (!block.contains(name.c_str()) && (allow.empty() || allow.contains(name.c_str()))) {
uint16_t which = event_struct.getFieldByName(name).getProto().getDiscriminantValue();
bool in_block = std::find(block.begin(), block.end(), name) != block.end();
bool in_allow = std::find(allow.begin(), allow.end(), name) != allow.end();
if (!in_block && (allow.empty() || in_allow)) {
uint16_t which = event_schema.getFieldByName(name).getProto().getDiscriminantValue();
sockets_[which] = name.c_str();
active_services.push_back(name);
}
}
if (!allow.isEmpty()) {

if (!allow.empty()) {
for (int i = 0; i < sockets_.size(); ++i) {
filters_.push_back(i == cereal::Event::Which::INIT_DATA || i == cereal::Event::Which::CAR_PARAMS || sockets_[i]);
}
}

std::vector<const char *> s;
std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(s),
[](const char *name) { return name != nullptr; });
qDebug() << "services " << s;
qDebug() << "loading route " << route;
rInfo("active services: %s", join(active_services, ',').c_str());
rInfo("loading route %s", route.c_str());

if (sm == nullptr) {
pm = std::make_unique<PubMaster>(s);
std::vector<const char *> socket_names;
std::copy_if(sockets_.begin(), sockets_.end(), std::back_inserter(socket_names),
[](const char *name) { return name != nullptr; });
pm = std::make_unique<PubMaster>(socket_names);
}
route_ = std::make_unique<Route>(route, data_dir);
}
Expand Down Expand Up @@ -68,23 +75,23 @@ void Replay::stop() {

bool Replay::load() {
if (!route_->load()) {
qCritical() << "failed to load route" << route_->name()
<< "from" << (route_->dir().isEmpty() ? "server" : route_->dir());
rError("failed to load route %s from %s", route_->name().c_str(),
route_->dir().empty() ? "server" : route_->dir().c_str());
return false;
}

for (auto &[n, f] : route_->segments()) {
bool has_log = !f.rlog.isEmpty() || !f.qlog.isEmpty();
bool has_video = !f.road_cam.isEmpty() || !f.qcamera.isEmpty();
bool has_log = !f.rlog.empty() || !f.qlog.empty();
bool has_video = !f.road_cam.empty() || !f.qcamera.empty();
if (has_log && (has_video || hasFlag(REPLAY_FLAG_NO_VIPC))) {
segments_.insert({n, nullptr});
}
}
if (segments_.empty()) {
qCritical() << "no valid segments in route" << route_->name();
rInfo("no valid segments in route: %s", route_->name().c_str());
return false;
}
rInfo("load route %s with %zu valid segments", qPrintable(route_->name()), segments_.size());
rInfo("load route %s with %zu valid segments", route_->name().c_str(), segments_.size());
max_seconds_ = (segments_.rbegin()->first + 1) * 60;
return true;
}
Expand Down Expand Up @@ -167,7 +174,7 @@ void Replay::buildTimeline() {
const auto &route_segments = route_->segments();
for (auto it = route_segments.cbegin(); it != route_segments.cend() && !exit_; ++it) {
std::shared_ptr<LogReader> log(new LogReader());
if (!log->load(it->second.qlog.toStdString(), &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue;
if (!log->load(it->second.qlog, &exit_, !hasFlag(REPLAY_FLAG_NO_FILE_CACHE), 0, 3) || log->events.empty()) continue;

std::vector<std::tuple<double, double, TimelineType>> timeline;
for (const Event &e : log->events) {
Expand Down
Loading

0 comments on commit d7051c4

Please sign in to comment.