Added option -j --json to retdec-bin2pat.

The added option -j --json is being used to pass the list of objects
from retdec-signature-from-library-creator.py as a json file.
Resolves avast#472

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 # dev
 
+* Fix: Added option to `retdec-bin2pat` to have the objects list passed through a json file.
+Modified `retdec-signature-from-library-creator.py` to pass objects list as a json file. ([#472](https://github.com/avast-tl/retdec/issues/472)).
 * New Feature: Added presentation of imported types and TypeRef hashes for .NET binaries ([#363](https://github.com/avast-tl/retdec/issues/363), [#364](https://github.com/avast-tl/retdec/issues/364), [#428](https://github.com/avast-tl/retdec/issues/428)).
 * New Feature: Added computation and presentation of icon hashes for exact and also similarity matching in PE files ([#339](https://github.com/avast-tl/retdec/issues/339)).
 * Enhancement: Added support for build and run on FreeBSD and potentially on other BSD OSes ([#476](https://github.com/avast-tl/retdec/pull/476)).

scripts/retdec-signature-from-library-creator.py

@@ -10,6 +10,7 @@
 import os
 import subprocess
 import tempfile
+from json import dump
 
 import importlib
 config = importlib.import_module('retdec-config')
@@ -72,6 +73,7 @@ def __init__(self, _args):
     def print_error_and_cleanup(self, message):
         if not self.args.no_cleanup:
             shutil.rmtree(self.tmp_dir_path, ignore_errors=True)
+            os.remove(self.tmp_file_path)
         utils.print_error(message)
 
     def _check_arguments(self):
@@ -82,6 +84,7 @@ def _check_arguments(self):
 
         dir_name = os.path.dirname(os.path.abspath(self.args.output))
         self.tmp_dir_path = tempfile.mkdtemp(dir=dir_name)
+        self.tmp_fd, self.tmp_file_path = tempfile.mkstemp(dir=dir_name, text=True)
 
         if self.args.ignore_nops:
             self.ignore_nop = '--ignore-nops'
@@ -126,7 +129,9 @@ def run(self):
             # Extract patterns from library.
             pattern_file = os.path.join(self.tmp_dir_path, lib_name) + '.pat'
             pattern_files.append(pattern_file)
-            _, result, _ = cmd.run_cmd([config.BIN2PAT, '-o', pattern_file] + objects, discard_stdout=True, discard_stderr=True)
+            with open(self.tmp_fd, 'w') as tmp_fp:
+                dump(objects, tmp_fp)
+            _, result, _ = cmd.run_cmd([config.BIN2PAT, '-o', pattern_file, '-j', self.tmp_file_path], discard_stdout=True, discard_stderr=True)
 
             if result != 0:
                 self.print_error_and_cleanup('utility bin2pat failed when processing %s' % lib_path)
@@ -161,6 +166,7 @@ def run(self):
         # Do cleanup.
         if not self.args.no_cleanup:
             shutil.rmtree(self.tmp_dir_path, ignore_errors=True)
+            os.remove(self.tmp_file_path)
 
         return result
 

src/bin2pat/bin2pat.cpp

@@ -9,6 +9,7 @@
 #include <iostream>
 #include <ostream>
 #include <vector>
+#include <json/json.h>
 
 #include "retdec/utils/filesystem_path.h"
 #include "retdec/patterngen/pattern_extractor/pattern_extractor.h"
@@ -28,13 +29,16 @@ void printUsage(
 	std::ostream &outputStream)
 {
 	outputStream << "Usage: bin2pat [-o OUTPUT_FILE] [-n NOTE]"
-		<< " INPUT_FILE [INPUT_FILE...]\n\n"
+		<< " <INPUT_FILE [INPUT_FILE...] | JSON_FILE>\n\n"
 		<< "-o --output OUTPUT_FILE\n"
 		<< "    Output file path (if not given, stdout is used).\n"
 		<< "    If multiple paths are given, only last one is used.\n\n"
 		<< "-n --note NOTE\n"
 		<< "    Optional note that will be added to all rules.\n"
-		<< "    If multiple notes are given, only last one is used.\n\n";
+		<< "    If multiple notes are given, only last one is used.\n\n"
+		<< "-j --json JSON_FILE\n"
+		<< "    Optionally pass the list of input files as a json file\n"
+		<< "    This is useful for a large number of input files.\n\n";
 }
 
 void printErrorAndDie(
@@ -81,6 +85,30 @@ void processArgs(
 				return;
 			}
 		}
+		else if (args[i] == "-j" || args[i] == "--json") {
+			Json::Value jsonIn;
+			Json::Reader reader;
+			std::ifstream jsonFile(args[++i], std::ifstream::binary);
+			if (!jsonFile) {
+				printErrorAndDie("JSON_FILE '" + args[i]
+					+ "' does not exist");
+				return;
+			}
+			else {
+				if (reader.parse(jsonFile, jsonIn, false))
+				{
+					for (int j = 0; j < jsonIn.size(); j++)
+						inPaths.push_back(jsonIn[j].asCString());
+					jsonFile.close();
+				}
+				else
+				{
+					jsonFile.close();
+					printErrorAndDie("JSON_FILE '" + args[i]
+					+ "' could not be parsed");
+				}
+			}
+		}
 		else {
 			// Input file. Check file on system level.
 			if(!FilesystemPath(args[i]).isFile()) {