Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

METRON-2306: [UI] Grok parsers' have duplicate timestampField #1553

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

METRON-2306: [UI] Grok parsers' have duplicate timestampField #1553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1e818b6
remove duplicate timestampField input
sardell Nov 2, 2019
57688e9
fix parserConfig type
sardell Nov 4, 2019
04d94aa
update unit tests for parser config component
sardell Nov 4, 2019
8afa263
add validation to timestampField input
sardell Nov 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions metron-interface/metron-config/src/app/model/sensor-parser-config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export class SensorParserConfig {
writerClassName: string;
errorWriterClassName: string;
invalidWriterClassName: string;
parserConfig: {};
parserConfig: { [key: string]: any; }
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typing using {} just extends the Object interface. What we really want here is an object literal that's editable by users.

fieldTransformations: FieldTransformer[];
numWorkers: number;
numAckers: number;
Expand All @@ -35,13 +35,11 @@ export class SensorParserConfig {
errorWriterNumTasks: number;
spoutConfig: {};
stormConfig: {};
timestampField: string;

constructor() {
this.parserConfig = {};
this.fieldTransformations = [];
this.spoutConfig = {};
this.stormConfig = {};
this.timestampField = 'timestamp';
}
}
8 changes: 1 addition & 7 deletions ...ce/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,6 @@
</div>
</div>

<div class="form-group" *ngIf="isGrokParser(sensorParserConfig)" data-qe-id="timestamp-field">
<label attr.for="timestampField">TIMESTAMP *</label>
<input type="text" class="form-control" name="timestampField" formControlName="timestampField" [(ngModel)]="sensorParserConfig.timestampField">
<label *ngIf="!sensorParserConfig.timestampField"><span class="warning-text">Field is required. </span></label>
</div>

<div class="form-group" [ngClass]="{'panel-selected': showFieldSchema }" >
<label attr.for="fieldSchema">SCHEMA</label>
<div class="input-group" [attr.disabled]="!isConfigValid()">
Expand Down Expand Up @@ -195,7 +189,7 @@
</div>
<div class="form-group">
<label attr.for="parserConfig">PARSER CONFIG</label>
<metron-config-advanced-form name="parserConfig" [(config)]="sensorParserConfig.parserConfig"></metron-config-advanced-form>
<metron-config-advanced-form name="parserConfig" [(config)]="sensorParserConfig.parserConfig" [parserClassName]="sensorParserConfig.parserClassName"></metron-config-advanced-form>
</div>

</div>
Expand Down
14 changes: 9 additions & 5 deletions ...metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -583,7 +583,7 @@ describe('Component: SensorParserConfig', () => {
);
component.createForms();

expect(Object.keys(component.sensorConfigForm.controls).length).toEqual(17);
expect(Object.keys(component.sensorConfigForm.controls).length).toEqual(16);
expect(
Object.keys(component.transformsValidationForm.controls).length
).toEqual(2);
Expand Down Expand Up @@ -821,6 +821,7 @@ describe('Component: SensorParserConfig', () => {
expect(component.isConfigValid()).toEqual(false);

component.grokStatement = 'grok statement';
component.sensorParserConfig.parserConfig.timestampField = 'timestamp';
expect(component.isConfigValid()).toEqual(true);
}));

Expand Down Expand Up @@ -1119,8 +1120,11 @@ describe('Component: SensorParserConfig', () => {
expect(component.showAdvancedParserConfiguration).toEqual(false);
}));

it('should be timestamp by default', () => {
expect(component.sensorParserConfig.timestampField).toEqual('timestamp');
it('should be timestamp by default when parser is Grok', () => {
component.sensorParserConfig.parserClassName = 'org.apache.metron.parsers.GrokParser';
component.paramsID = 'new';
component.onParserTypeChange();
expect(component.sensorParserConfig.parserConfig.timestampField).toEqual('timestamp');
});

it('should be invalid if timestamp field is empty', () => {
Expand All @@ -1130,9 +1134,9 @@ describe('Component: SensorParserConfig', () => {
component.grokStatement = 'grokStatement';
component.sensorParserConfig = new SensorParserConfig();
component.sensorParserConfig.parserClassName = 'org.apache.metron.parsers.GrokParser'
component.sensorParserConfig.timestampField = '';
component.sensorParserConfig.parserConfig.timestampField = '';
expect(component.isConfigValid()).toEqual(false);
component.sensorParserConfig.timestampField = 'timestamp';
component.sensorParserConfig.parserConfig.timestampField = 'timestamp';
expect(component.isConfigValid()).toEqual(true);
});
});
16 changes: 9 additions & 7 deletions ...face/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ export class SensorParserConfigComponent implements OnInit {
grokStatement = '';
patternLabel = '';
currentSensors = [];
paramsID: string;

editMode: boolean = false;

Expand Down Expand Up @@ -191,8 +192,8 @@ export class SensorParserConfigComponent implements OnInit {

ngOnInit() {
this.route.params.subscribe(params => {
let id = params['id'];
this.init(id);
this.paramsID = params['id'];
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Storing the id in a class property allows me to use it when evaluating whether or not to populate the parserConfig object with the timestampField property and default timestamp value.

this.init(this.paramsID);
});
this.createForms();
this.getAvailableParsers();
Expand All @@ -210,10 +211,6 @@ export class SensorParserConfigComponent implements OnInit {
this.sensorParserConfig.parserClassName,
Validators.required
);
group['timestampField'] = new FormControl(
this.sensorParserConfig.timestampField,
Validators.required
);
group['grokStatement'] = new FormControl(this.grokStatement);
group['transforms'] = new FormControl(
this.sensorParserConfig['transforms']
Expand Down Expand Up @@ -317,6 +314,11 @@ export class SensorParserConfigComponent implements OnInit {
}

onParserTypeChange(): void {
if (this.paramsID === 'new' && this.isGrokParser(this.sensorParserConfig)) {
this.sensorParserConfig.parserConfig.timestampField = 'timestamp';
} else if (this.paramsID === 'new') {
delete this.sensorParserConfig.parserConfig.timestampField;
}
this.parserClassValid =
this.sensorParserConfig.parserClassName !== undefined &&
this.sensorParserConfig.parserClassName.length > 0;
Expand All @@ -338,7 +340,7 @@ export class SensorParserConfigComponent implements OnInit {
this.kafkaTopicValid &&
this.parserClassValid &&
(!isGrokParser || this.isGrokStatementValid()) &&
(!isGrokParser || !!this.sensorParserConfig.timestampField);
(!isGrokParser || !!this.sensorParserConfig.parserConfig.timestampField);
}

getKafkaStatus() {
Expand Down
15 changes: 12 additions & 3 deletions ...ace/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,25 @@
<form role="form" [formGroup]="configForm">
<div *ngFor="let key of configKeys">
<div class="row mx-0 configkey-row">
<div class="col-md-10 advanced-input"><input type="text" class="form-control" name="advanced1" value="{{key}}" readonly></div>
<div class="col-md-10 advanced-input">
<input type="text" class="form-control" name="advanced1" value="{{key}}" readonly>
</div>
<div class="col-md-2">
</div>
</div>
<div class="row mx-0">
<div class="col-md-10 advanced-input"><input type="text" class="form-control" formControlName="{{key}}" [ngModel]="displayValue(key)" (ngModelChange)="saveValue(key, $event)"></div>
<div class="col-md-2" (click)="removeConfig(key)">
<div class="col-md-10 advanced-input">
<input type="text" class="form-control" formControlName="{{key}}" [ngModel]="displayValue(key)" (ngModelChange)="saveValue(key, $event)">
</div>
<div class="col-md-2" (click)="removeConfig(key)" *ngIf="key !== 'timestampField'">
<i class="fa fa-minus fa-4 icon-button" aria-hidden="true" ></i>
</div>
</div>
<div *ngIf="isGrokParser() && key === 'timestampField'">
<label *ngIf="!config['timestampField']">
<span class="warning-text">Field is required. </span>
</label>
</div>
</div>
<div class="row mx-0 new-configkey-row">
<div class="col-md-10 advanced-input"><input type="text" class="form-control" (focus)="clearKeyPlaceholder()" (blur)="saveNewConfig()" [ngClass]="{'input-placeholder': newConfigKey == 'enter field'}" formControlName="newConfigKey" [(ngModel)]="newConfigKey"></div>
Expand Down
5 changes: 5 additions & 0 deletions ...rface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {FormGroup, Validators, FormControl} from '@angular/forms';
export class AdvancedConfigFormComponent implements OnInit, OnChanges {

@Input() config: {};
@Input() parserClassName: string;
configKeys: string[] = [];
newConfigKey: string = 'enter field';
newConfigValue: string = 'enter value';
Expand Down Expand Up @@ -125,4 +126,8 @@ export class AdvancedConfigFormComponent implements OnInit, OnChanges {

}

isGrokParser(): boolean {
return this.parserClassName === 'org.apache.metron.parsers.GrokParser';
}

}