Added RosPackageFilters & blocklist.json (#891)

On windows workflow use node 18.12.0 inplace of 18.13.0. There is a repeatable issue with node-gyp configuration on node 18.13. It seems to be related to the node cache. Switching to 18.12 avoids using a cached version of node 18 and the issue no longer occurs. Fix #890
RobotWebTools · Apr 3, 2023 · 06d1646 · 06d1646
1 parent a14527b
commit 06d1646
Show file tree

Hide file tree

Showing 7 changed files with 162 additions and 42 deletions.
diff --git a/.github/workflows/windows-build-and-test-compatibility.yml b/.github/workflows/windows-build-and-test-compatibility.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.X, 19.X]
+        node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.12.X, 19.X]
         ros_distribution:
           # - foxy
           - galactic

diff --git a/.github/workflows/windows-build-and-test.yml b/.github/workflows/windows-build-and-test.yml
@@ -26,7 +26,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.X, 19.X]
+        node-version: [10.X, 12.X, 14.X, 16.11.X, 17.X, 18.12.0, 19.X]
     steps:
     - name: Setup Node.js ${{ matrix.node-version }}
       uses: actions/setup-node@v2

diff --git a/rosidl_gen/blocklist.json b/rosidl_gen/blocklist.json
@@ -0,0 +1,5 @@
+[
+  {
+    "pkgName": "rosbag2_storage_mcap_testdata"
+  }
+]
diff --git a/rosidl_gen/filter.js b/rosidl_gen/filter.js
@@ -0,0 +1,104 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+// blocklist.json format
+//  [
+//    {
+//      pkgName: RegExString,
+//      interfaceName: RegExString,
+//      os: RegExString
+//    },
+//    ...
+//  ]
+//
+// examples
+//  [
+//    {
+//      "pkgName": "action*"
+//    },
+//    {
+//      "pkgName": "std_msgs",
+//    },
+//    {
+//      "pkgName": "std_msgs",
+//      "interfaceName": "String"
+//    },
+//    {
+//      "os": "Linux"
+//    },
+//  ]
+
+const RosPackageFilters = {
+  filters: [],
+  _loaded: false,
+
+  addFilter: function (pkgName, interfaceName, os) {
+    this.filters.push({
+      pkgName: pkgName,
+      interfaceName: interfaceName,
+      os: os,
+    });
+  },
+
+  _matches: function (filter, pkgInfo) {
+    if (filter.os && filter.os.test(os.type())) {
+      return true;
+    }
+
+    if (filter.pkgName) {
+      if (filter.pkgName.test(pkgInfo.pkgName)) {
+        if (!filter.interfaceName) {
+          return true;
+        }
+      } else {
+        return false;
+      }
+    }
+
+    if (
+      filter.interfaceName &&
+      filter.interfaceName.test(pkgInfo.interfaceName)
+    ) {
+      return true;
+    }
+
+    return false;
+  },
+
+  load: function (
+    blocklistPath = path.join(__dirname, '../rosidl_gen/blocklist.json')
+  ) {
+    this._loaded = true;
+
+    if (!fs.existsSync(blocklistPath)) return;
+
+    // eslint-disable-next-line
+    let blocklistData = JSON.parse(fs.readFileSync(blocklistPath, 'utf8'));
+
+    let filters = blocklistData.map((pkgFilterData) => {
+      let filter = {};
+      if (pkgFilterData['pkgName']) {
+        filter.pkgName = new RegExp(pkgFilterData.pkgName);
+      }
+      if (pkgFilterData['interfaceName']) {
+        filter.interfaceName = new RegExp(pkgFilterData.interfaceName);
+      }
+      if (pkgFilterData['os']) {
+        filter.os = new RegExp(pkgFilterData.os);
+      }
+      return filter;
+    });
+
+    this.filters = filters.filter(
+      (filter) => !filter.os || filter.os.test(os.type())
+    );
+  },
+
+  matchesAny: function (pkgInfo) {
+    if (!this._loaded) this.load();
+    return this.filters.some((filter) => this._matches(filter, pkgInfo));
+  },
+};
+
+module.exports = RosPackageFilters;
diff --git a/rosidl_gen/index.js b/rosidl_gen/index.js
@@ -26,7 +26,9 @@ const installedPackagePaths = process.env.AMENT_PREFIX_PATH.split(
 
 async function generateInPath(path) {
   const pkgs = await packages.findPackagesInDirectory(path);
+
   const pkgsInfo = Array.from(pkgs.values());
+
   await Promise.all(
     pkgsInfo.map((pkgInfo) => generateJSStructFromIDL(pkgInfo, generatedRoot))
   );
@@ -42,6 +44,7 @@ async function generateAll(forcedGenerating) {
       path.join(__dirname, 'generator.json'),
       path.join(generatedRoot, 'generator.json')
     );
+
     await Promise.all(
       installedPackagePaths.map((path) => generateInPath(path))
     );

diff --git a/rosidl_gen/packages.js b/rosidl_gen/packages.js
@@ -21,6 +21,7 @@ const path = require('path');
 const walk = require('walk');
 const os = require('os');
 const flat = require('array.prototype.flat');
+const pkgFilters = require('../rosidl_gen/filter.js');
 
 const fsp = fs.promises;
 
@@ -140,28 +141,30 @@ async function findAmentPackagesInDirectory(dir) {
     pkgs.map((pkg) => getPackageDefinitionsFiles(pkg, dir))
   );
 
-  // Support flat() methond for nodejs < 11.
+  // Support flat() method for nodejs < 11.
   const rosFiles = Array.prototype.flat ? files.flat() : flat(files);
 
   const pkgMap = new Map();
   return new Promise((resolve, reject) => {
     rosFiles.forEach((filePath) => {
-      if (path.extname(filePath) === '.msg') {
-        // Some .msg files were generated prior to 0.3.2 for .action files,
-        // which has been disabled. So these files should be ignored here.
-        if (path.dirname(dir).split(path.sep).pop() !== 'action') {
-          addInterfaceInfo(
-            grabInterfaceInfo(filePath, true),
-            'messages',
-            pkgMap
-          );
-        }
-      } else if (path.extname(filePath) === '.srv') {
-        addInterfaceInfo(grabInterfaceInfo(filePath, true), 'services', pkgMap);
-      } else if (path.extname(filePath) === '.action') {
-        addInterfaceInfo(grabInterfaceInfo(filePath, true), 'actions', pkgMap);
+      const interfaceInfo = grabInterfaceInfo(filePath, true);
+      const ignore = pkgFilters.matchesAny(interfaceInfo);
+      if (ignore) {
+        console.log('Omitting filtered interface: ', interfaceInfo);
       } else {
-        // we ignore all other files
+        if (path.extname(filePath) === '.msg') {
+          // Some .msg files were generated prior to 0.3.2 for .action files,
+          // which has been disabled. So these files should be ignored here.
+          if (path.dirname(dir).split(path.sep).pop() !== 'action') {
+            addInterfaceInfo(interfaceInfo, 'messages', pkgMap);
+          }
+        } else if (path.extname(filePath) === '.srv') {
+          addInterfaceInfo(interfaceInfo, 'services', pkgMap);
+        } else if (path.extname(filePath) === '.action') {
+          addInterfaceInfo(interfaceInfo, 'actions', pkgMap);
+        } else {
+          // we ignore all other files
+        }
       }
     });
     resolve(pkgMap);
@@ -191,30 +194,27 @@ async function findPackagesInDirectory(dir) {
       let walker = walk.walk(dir, { followLinks: true });
       let pkgMap = new Map();
       walker.on('file', (root, file, next) => {
-        if (path.extname(file.name) === '.msg') {
-          // Some .msg files were generated prior to 0.3.2 for .action files,
-          // which has been disabled. So these files should be ignored here.
-          if (path.dirname(root).split(path.sep).pop() !== 'action') {
-            addInterfaceInfo(
-              grabInterfaceInfo(path.join(root, file.name), amentExecuted),
-              'messages',
-              pkgMap
-            );
-          }
-        } else if (path.extname(file.name) === '.srv') {
-          addInterfaceInfo(
-            grabInterfaceInfo(path.join(root, file.name), amentExecuted),
-            'services',
-            pkgMap
-          );
-        } else if (path.extname(file.name) === '.action') {
-          addInterfaceInfo(
-            grabInterfaceInfo(path.join(root, file.name), amentExecuted),
-            'actions',
-            pkgMap
-          );
+        const interfaceInfo = grabInterfaceInfo(
+          path.join(root, file.name),
+          amentExecuted
+        );
+        const ignore = pkgFilters.matchesAny(interfaceInfo);
+        if (ignore) {
+          console.log('Omitting filtered interface: ', interfaceInfo);
         } else {
-          // we ignore all other files
+          if (path.extname(file.name) === '.msg') {
+            // Some .msg files were generated prior to 0.3.2 for .action files,
+            // which has been disabled. So these files should be ignored here.
+            if (path.dirname(root).split(path.sep).pop() !== 'action') {
+              addInterfaceInfo(interfaceInfo, 'messages', pkgMap);
+            }
+          } else if (path.extname(file.name) === '.srv') {
+            addInterfaceInfo(interfaceInfo, 'services', pkgMap);
+          } else if (path.extname(file.name) === '.action') {
+            addInterfaceInfo(interfaceInfo, 'actions', pkgMap);
+          } else {
+            // we ignore all other files
+          }
         }
         next();
       });

diff --git a/rostsd_gen/index.js b/rostsd_gen/index.js
@@ -31,6 +31,7 @@ declare module "rclnodejs" {
 const path = require('path');
 const fs = require('fs');
 const loader = require('../lib/interface_loader.js');
+const pkgFilters = require('../rosidl_gen/filter.js');
 
 async function generateAll() {
   // load pkg and interface info (msgs and srvs)
@@ -63,7 +64,14 @@ function getPkgInfos(rootDir) {
 
     for (let filename of files) {
       const typeClass = fileName2Typeclass(filename);
-      if (!typeClass.type) continue;
+      if (
+        !typeClass.type ||
+        pkgFilters.matchesAny({
+          pkgName: typeClass.package,
+          interfaceName: typeClass.name,
+        })
+      )
+        continue;
 
       const rosInterface = loader.loadInterface(typeClass);