Skip to content

Commit

Permalink
Merge pull request #54 from gjgiezeman/json1-communication
Browse files Browse the repository at this point in the history
Changes in order to let the check command work with the json-server
  • Loading branch information
slingerbv authored Sep 5, 2024
2 parents 40818df + f31f80e commit 5293c2e
Show file tree
Hide file tree
Showing 7 changed files with 442 additions and 149 deletions.
91 changes: 73 additions & 18 deletions src/Command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import DatabaseRequest from './DatabaseRequest';
import { ProjectMetadata } from './modules/searchSECO-crawler/src/Crawler';
import MatchPrinter from './Print';
import config from './config/config';
import { CheckResponse, JsonRequest, ProjectInfoResponseItem, ProjectWithVersion } from './JsonRequest';

/**
* Makes a designated repo download location for the current miner.
Expand Down Expand Up @@ -98,26 +99,74 @@ export default abstract class Command {

/**
* Parses a project and retrieves author data.
* AuthorData is a map from filenames to CodeBlocks
* @returns a tuple containing a HashData array and an AuthorData object
*/
protected async parseAndBlame(): Promise<[HashData[], AuthorData]> {
const [filenames, hashes] = await this._moduleFacade.ParseRepository();
const [filenames, methods] = await this._moduleFacade.ParseRepository();
if (methods.length == 0) {
Logger.Debug('No methods found, skipping authors', Logger.GetCallerLocation());
return [methods, new Map() as AuthorData];
}
// Select the files where we found a method
const filteredFileNames: string[] = [];

hashes.forEach((hash) => {
methods.forEach((hash) => {
const idx = filenames.findIndex((file) => file === hash.FileName);
if (idx < 0) return;
filteredFileNames.push(filenames[idx]);
filenames.splice(idx, 1);
});
const authorData = await this._moduleFacade.GetAuthors(filteredFileNames);
return [methods, authorData];
}

if (hashes.length == 0) {
Logger.Debug('No methods found, skipping authors', Logger.GetCallerLocation());
return [hashes, new Map() as AuthorData];

static add_if_new(pid: number, version: number, pv_map: Map<number, number[]>) {
let pid_list = pv_map.get(pid);
if (!pid_list) {
pv_map.set(pid, [version]);
} else if (!pid_list.includes(version)) {
pid_list.push(version);
}

const authorData = await this._moduleFacade.GetAuthors(filteredFileNames);
return [hashes, authorData];
}

static get_project_versions(methods: CheckResponse[]): ProjectWithVersion[] {
let map: Map<number, number[]> = new Map();
methods.forEach(m => {
if (m.sv_time) {
this.add_if_new(m.pid, m.sv_time, map);
}
if (m.ev_time) {
this.add_if_new(m.pid, m.ev_time, map);
}
})
let result: ProjectWithVersion[] = [];
map.forEach((vs, k) => {
vs.forEach((v) => result.push({ project_id: k, version: v }));
});
return result;
}

static get_project_authors(methods: CheckResponse[]): Set<string> {
let authors: Set<string> = new Set();
methods.forEach(m => {
m.authors.forEach(a => authors.add(a))
})
return authors;
}

static order_project_info(pi_items: ProjectInfoResponseItem[]): Map<number, ProjectInfoResponseItem[]> {
let result = new Map<number, ProjectInfoResponseItem[]>();
pi_items.forEach((pi) => {
let pi_list = result.get(pi.pid);
if (!pi_list) {
result.set(pi.pid, [pi]);
} else {
pi_list.push(pi);
}
})
return result;
}

protected async checkProject(): Promise<boolean> {
Expand All @@ -127,19 +176,25 @@ export default abstract class Command {
const metadata = await this._moduleFacade.GetProjectMetadata(url);
if (!metadata) return false;

if (!this._flags.Branch)
if (!this._flags.Branch)
this._flags.Branch = metadata.defaultBranch;

await this._moduleFacade.DownloadRepository(url, this._flags.Branch);

if (this._flags.ProjectCommit !== '')
if (this._flags.ProjectCommit !== '')
await this._moduleFacade.SwitchVersion(this._flags.ProjectCommit);

const [hashes, authorData] = await this.parseAndBlame();
const databaseResponse = await DatabaseRequest.FindMatches(hashes);
const [projectMethods, projectBlaming] = await this.parseAndBlame();
const projectHashes = Array.from(new Set<string>(projectMethods.map((hash) => hash.Hash)));
const checkResponse: CheckResponse[] = await JsonRequest.FindMatches(projectHashes);
const dbProjectIds = Command.get_project_versions(checkResponse);
const dbProjectInfo = Command.order_project_info(await JsonRequest.GetProjectData(dbProjectIds));
const authorIds = Command.get_project_authors(checkResponse);
const dbAuthorInfo = await JsonRequest.GetAuthorData(authorIds.values());


const printer = new MatchPrinter();
await printer.PrintHashMatches(hashes, databaseResponse, authorData, url, metadata.id);
await printer.PrintHashMatches(url, metadata.id, projectMethods, projectBlaming, checkResponse, dbProjectInfo, dbAuthorInfo);
printer.Close();
return true
}
Expand Down Expand Up @@ -171,7 +226,7 @@ export default abstract class Command {
}

const success = await this._moduleFacade.DownloadRepository(this._flags.MandatoryArgument, this._flags.Branch);
if (!success)
if (!success)
return;
metadata.versionHash = await this._moduleFacade.GetCurrentVersion();

Expand All @@ -189,7 +244,7 @@ export default abstract class Command {
await this.uploadPartialProject(commit.commit, commit.lines, commit.vulnerability, metadata);
}

if (metadata.defaultBranch !== this._flags.Branch)
if (metadata.defaultBranch !== this._flags.Branch)
await this._moduleFacade.SwitchVersion(this._flags.Branch);
const tags = await this._moduleFacade.GetRepositoryTags();
const tagc = tags.length;
Expand Down Expand Up @@ -238,9 +293,9 @@ export default abstract class Command {
trimmedHashes.forEach((hash) => {
filteredFileNames.push(
filenames[
filenames.findIndex((file) => {
file.includes(hash.FileName);
})
filenames.findIndex((file) => {
file.includes(hash.FileName);
})
]
);
});
Expand Down
13 changes: 13 additions & 0 deletions src/Input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ type Input = {
miner_name?: string;
github_token?: string;
wallet_address?: string;
json_token?: string;
json_url?: string;
_: (string | number)[];
$0: string;
};
Expand Down Expand Up @@ -96,6 +98,15 @@ export class InputParser {
description: 'The github token to use for downloading repositories',
alias: 'g',
})
.option('json_token', {
type: 'string',
description: 'The json token to use for authenticating in database requests',
})
.option('json_url', {
type: 'string',
description: 'The url where the json api to the database is located',
alias: 'u',
})
.option('wallet_address', {
type: 'string',
description: 'The wallet address used to link the miner to the DAO.',
Expand Down Expand Up @@ -149,6 +160,8 @@ export class InputParser {
if (parsed.miner_name) setInConfig('MINER_NAME', parsed.miner_name);
if (parsed.github_token) setInConfig('GITHUB_TOKEN', parsed.github_token);
if (parsed.wallet_address) setInConfig('PERSONAL_WALLET_ADDRESS', parsed.wallet_address);
if (parsed.json_token) setInConfig('JSON_TOKEN', parsed.json_token);
if (parsed.json_url) setInConfig('JSON_URL', parsed.json_url);

setInConfig('COMMAND', command);

Expand Down
169 changes: 169 additions & 0 deletions src/JsonRequest.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
import HashData from './modules/searchSECO-parser/src/HashData';
import Logger from './modules/searchSECO-logger/src/Logger';
import config from './config/config'


const dbapi_url = config.JSON_URL;

const token = config.JSON_TOKEN;


interface CheckRequest {
Check: string[];
}

interface MethodInFile {
file: string;
line: number;
pid: string;
project_versions: number[];
}

export type CheckResponse = {
mh: string;
pid: number;
method: string;
sv_time: number;
sv_hash: string;
ev_time: number;
ev_hash: string;
file: string;
line: number;
pv: number;
vuln: string | null;
authors: string[];
};

export interface ProjectWithVersion {
project_id: number;
version: number;
}

interface ProjectInfoRequest {
ProjectInfo: ProjectWithVersion[];
}

export type ProjectInfoResponseItem = {
pid: number,
vtime: number,
vhash: string,
license: string,
name: string,
url: string,
oid: string,
pv: number,
hashes?: string[]
}

interface AuthorRequest {
AuthorInfo: string[];
}

export type AuthorInfoResponseItem = {
name: string,
mail: string,
id: string
}

type Content =
CheckRequest | ProjectInfoRequest | AuthorRequest;


interface Request {
token: string;
content: Content;
}

export class JsonRequest {

static async PerformRequest(content: Content): Promise<Object> {
let req: Request = {
'token': token,
'content': content
}
const response: Response = await fetch(dbapi_url, {
method: 'POST',
body: JSON.stringify(req),
headers: {
'Content-Type': 'application/json',
Accept: 'application/json',
},
});

if (!response.ok) {
throw new Error(`Error! status: ${response.status}`);
}

const obj: Object = (await response.json());
return obj;
}

public static async FindMatches(hashes: string[]): Promise<CheckResponse[]> {
Logger.Debug(`In FindMatches (JsonRequest)`, Logger.GetCallerLocation());
let content: Content = {
'Check': hashes
}
try {
let obj = await this.PerformRequest(content);
const pr = obj as CheckResponse[];
if (pr.length > 0) {
const md0 = JSON.stringify(pr[0], null, 4);
Logger.Debug(`First CheckResponse: ${md0}`, Logger.GetCallerLocation());
}
return pr;
} catch (error) {
if (error instanceof Error) {
Logger.Warning(`error message: ${error.message}`, Logger.GetCallerLocation());
throw error;
} else {
Logger.Warning(`unexpected error: $error`, Logger.GetCallerLocation());
throw 'An unexpected error occurred';
}
}
Logger.Debug(`First CheckResponse returning empty result`, Logger.GetCallerLocation());
return [];
}



public static async GetProjectData(pireq: ProjectWithVersion[]): Promise<ProjectInfoResponseItem[]> {
let content: Content = {
'ProjectInfo': pireq
}
try {
let obj = await this.PerformRequest(content);
let result = obj as ProjectInfoResponseItem[];
return result;
} catch (error) {
if (error instanceof Error) {
Logger.Warning(`error message: ${error.message}`, Logger.GetCallerLocation());
throw error;
} else {
Logger.Warning(`unexpected error: $error`, Logger.GetCallerLocation());
throw 'An unexpected error occurred';
}
}
return []
}

public static async GetAuthorData(authorIds: Iterable<string>): Promise<AuthorInfoResponseItem[]> {
const authorArray = Array.from(authorIds)
let content: Content = { 'AuthorInfo': authorArray };
try {
let obj = await this.PerformRequest(content);
let result = obj as AuthorInfoResponseItem[];
return result;
} catch (error) {
if (error instanceof Error) {
Logger.Warning(`error message: ${error.message}`, Logger.GetCallerLocation());
throw error;
} else {
Logger.Warning(`unexpected error: $error`, Logger.GetCallerLocation());
throw 'An unexpected error occurred';
}
}
return []
}

}

8 changes: 4 additions & 4 deletions src/ModuleFacade.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,11 +86,11 @@ export default class ModuleFacade {
}

/**
* Retrieves author data from a project, only retrieving data from specified files
* Retrieves author data from a project, only retrieving data from specified files
* @param files A list of files to fetch the author data for
* @returns The fetched author data
*/
* @param files A list of files to fetch the author data for
* @returns The fetched author data
*/
public async GetAuthors(files: string[]): Promise<AuthorData> {
Logger.Debug('Calling the spider to download author data', Logger.GetCallerLocation());
let authorData: AuthorData = new Map();
Expand Down
Loading

0 comments on commit 5293c2e

Please sign in to comment.