Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

NETOBSERV-1274 index duplicate (improve query perfs) #380

Merged
merged 2 commits into from
Sep 27, 2023
Merged

NETOBSERV-1274 index duplicate (improve query perfs) #380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
63b2688
NETOBSERV-1274 index duplicate (improve query perfs)
jotak Sep 7, 2023
c6f748b
Move Type from field to label
jotak Sep 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions pkg/loki/flow_query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,17 @@ func NewFlowQueryBuilder(cfg *Config, start, end, limit string, dedup bool,

lineFilters := []lineFilter{}
if dedup {
lineFilters = append(lineFilters, lineFilter{
key: fields.Duplicate,
values: []lineMatch{{
valueType: typeBool,
value: "false",
}},
})
if cfg.IsLabel(fields.Duplicate) {
labelFilters = append(labelFilters, stringEqualLabelFilter(fields.Duplicate, "false"))
} else {
lineFilters = append(lineFilters, lineFilter{
key: fields.Duplicate,
values: []lineMatch{{
valueType: typeBool,
value: "false",
}},
})
}
}

if packetLoss == constants.PacketLossDropped {
Expand Down
8 changes: 4 additions & 4 deletions web/src/api/ipfix.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ export interface Labels {
SrcK8S_OwnerName?: string;
/** Destination owner, such as Deployment, StatefulSet, etc. */
DstK8S_OwnerName?: string;
/** Kind of the source matched Kubernetes object, such as Pod, Service, etc. */
SrcK8S_Type?: string;
/** Kind of the destination matched Kubernetes object, such as Pod name, Service name, etc. */
DstK8S_Type?: string;
/** Flow direction from the node observation point */
FlowDirection: FlowDirection;
/** Type of record: 'flowLog' for regular flow logs, or 'allConnections',
Expand Down Expand Up @@ -48,10 +52,6 @@ export interface Fields {
SrcK8S_Name?: string;
/** Name of the destination matched Kubernetes object, such as Pod name, Service name, etc. */
DstK8S_Name?: string;
/** Kind of the source matched Kubernetes object, such as Pod, Service, etc. */
SrcK8S_Type?: string;
/** Kind of the destination matched Kubernetes object, such as Pod name, Service name, etc. */
DstK8S_Type?: string;
/** Source port */
SrcPort?: number;
/** Destination port */
Expand Down
16 changes: 8 additions & 8 deletions web/src/components/netflow-record/record-field.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,13 +235,13 @@ export const RecordField: React.FC<{
);
case ColumnsId.name:
return doubleContainer(
kubeObjContent(flow.fields.SrcK8S_Name, flow.fields.SrcK8S_Type, flow.labels.SrcK8S_Namespace),
kubeObjContent(flow.fields.DstK8S_Name, flow.fields.DstK8S_Type, flow.labels.DstK8S_Namespace)
kubeObjContent(flow.fields.SrcK8S_Name, flow.labels.SrcK8S_Type, flow.labels.SrcK8S_Namespace),
kubeObjContent(flow.fields.DstK8S_Name, flow.labels.DstK8S_Type, flow.labels.DstK8S_Namespace)
);
case ColumnsId.srcname:
return singleContainer(kubeObjContent(value as string, flow.fields.SrcK8S_Type, flow.labels.SrcK8S_Namespace));
return singleContainer(kubeObjContent(value as string, flow.labels.SrcK8S_Type, flow.labels.SrcK8S_Namespace));
case ColumnsId.dstname:
return singleContainer(kubeObjContent(value as string, flow.fields.DstK8S_Type, flow.labels.DstK8S_Namespace));
return singleContainer(kubeObjContent(value as string, flow.labels.DstK8S_Type, flow.labels.DstK8S_Namespace));
case ColumnsId.owner:
return doubleContainer(
kubeObjContent(flow.labels.SrcK8S_OwnerName, flow.fields.SrcK8S_OwnerType, flow.labels.SrcK8S_Namespace),
Expand Down Expand Up @@ -269,14 +269,14 @@ export const RecordField: React.FC<{
explicitKubeObjContent(
flow.fields.SrcAddr,
flow.fields.SrcPort || NaN,
flow.fields.SrcK8S_Type,
flow.labels.SrcK8S_Type,
flow.labels.SrcK8S_Namespace,
flow.fields.SrcK8S_Name
),
explicitKubeObjContent(
flow.fields.DstAddr,
flow.fields.DstPort || NaN,
flow.fields.DstK8S_Type,
flow.labels.DstK8S_Type,
flow.labels.DstK8S_Namespace,
flow.fields.DstK8S_Name
)
Expand All @@ -286,7 +286,7 @@ export const RecordField: React.FC<{
explicitKubeObjContent(
flow.fields.SrcAddr,
flow.fields.SrcPort || NaN,
flow.fields.SrcK8S_Type,
flow.labels.SrcK8S_Type,
flow.labels.SrcK8S_Namespace,
flow.fields.SrcK8S_Name
)
Expand All @@ -296,7 +296,7 @@ export const RecordField: React.FC<{
explicitKubeObjContent(
flow.fields.DstAddr,
flow.fields.DstPort || NaN,
flow.fields.DstK8S_Type,
flow.labels.DstK8S_Type,
flow.labels.DstK8S_Namespace,
flow.fields.DstK8S_Name
)
Expand Down
4 changes: 2 additions & 2 deletions web/src/components/netflow-record/record-panel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,15 +231,15 @@ export const RecordPanel: React.FC<RecordDrawerProps> = ({
columns.filter(
c =>
//remove empty / duplicates columns for Node
(record?.fields.SrcK8S_Type !== 'Node' ||
(record?.labels.SrcK8S_Type !== 'Node' ||
![
ColumnsId.srcnamespace,
ColumnsId.srcowner,
ColumnsId.srcownertype,
ColumnsId.srchostaddr,
ColumnsId.srchostname
].includes(c.id)) &&
(record?.fields.DstK8S_Type !== 'Node' ||
(record?.labels.DstK8S_Type !== 'Node' ||
![
ColumnsId.dstnamespace,
ColumnsId.dstowner,
Expand Down
8 changes: 4 additions & 4 deletions web/src/components/query-summary/summary-panel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,13 +165,13 @@ export const SummaryPanelContent: React.FC<{

if (flows && flows.length) {
//list all types
const types = Array.from(new Set(flows.flatMap(f => [f.fields.SrcK8S_Type, f.fields.DstK8S_Type])));
const types = Array.from(new Set(flows.flatMap(f => [f.labels.SrcK8S_Type, f.labels.DstK8S_Type])));
types
.filter((t: string | undefined) => t !== undefined)
.forEach((type: string) => {
const tc: TypeCardinality = { type, objects: [] };

const typeFilteredFlows = flows.filter(f => [f.fields.SrcK8S_Type, f.fields.DstK8S_Type].includes(type));
const typeFilteredFlows = flows.filter(f => [f.labels.SrcK8S_Type, f.labels.DstK8S_Type].includes(type));
//list all namespaces of this type
const typeNamespaces = new Set(
typeFilteredFlows.flatMap(f => [f.labels.SrcK8S_Namespace, f.labels.DstK8S_Namespace])
Expand All @@ -189,14 +189,14 @@ export const SummaryPanelContent: React.FC<{
//add all names of this namespace of type
namespaceFilteredFlows.forEach(record => {
const srcName =
record.fields.SrcK8S_Type === type && record.labels.SrcK8S_Namespace === namespace
record.labels.SrcK8S_Type === type && record.labels.SrcK8S_Namespace === namespace
? record.fields.SrcK8S_Name
: undefined;
if (srcName && !nsObject.names.includes(srcName)) {
nsObject.names.push(srcName);
}
const dstName =
record.fields.DstK8S_Type === type && record.labels.DstK8S_Namespace === namespace
record.labels.DstK8S_Type === type && record.labels.DstK8S_Namespace === namespace
? record.fields.DstK8S_Name
: undefined;
if (dstName && !nsObject.names.includes(dstName)) {
Expand Down
14 changes: 7 additions & 7 deletions web/src/utils/columns.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ export const getCommonColumns = (t: TFunction, withConcatenatedFields = true): C
id: ColumnsId.type,
name: t('Kinds'),
isSelected: false,
value: f => getSrcOrDstValue(f.fields.SrcK8S_Type, f.fields.DstK8S_Type),
value: f => getSrcOrDstValue(f.labels.SrcK8S_Type, f.labels.DstK8S_Type),
sort: (a, b, col) => compareStrings((col.value(a) as string[]).join(''), (col.value(b) as string[]).join('')),
width: 10
},
Expand Down Expand Up @@ -292,14 +292,14 @@ export const getCommonColumns = (t: TFunction, withConcatenatedFields = true): C
...getConcatenatedValue(
f.fields.SrcAddr,
f.fields.SrcPort || NaN,
f.fields.SrcK8S_Type,
f.labels.SrcK8S_Type,
f.labels.SrcK8S_Namespace,
f.fields.SrcK8S_Name
),
...getConcatenatedValue(
f.fields.DstAddr,
f.fields.DstPort || NaN,
f.fields.DstK8S_Type,
f.labels.DstK8S_Type,
f.labels.DstK8S_Namespace,
f.fields.DstK8S_Name
)
Expand Down Expand Up @@ -374,7 +374,7 @@ export const getSrcColumns = (t: TFunction): Column[] => {
fieldName: 'SrcK8S_Type',
quickFilter: 'src_kind',
isSelected: false,
value: f => f.fields.SrcK8S_Type || '',
value: f => f.labels.SrcK8S_Type || '',
sort: (a, b, col) => compareStrings(col.value(a) as string, col.value(b) as string),
width: 10
},
Expand Down Expand Up @@ -514,7 +514,7 @@ export const getDstColumns = (t: TFunction): Column[] => {
fieldName: 'DstK8S_Type',
quickFilter: 'dst_kind',
isSelected: false,
value: f => f.fields.DstK8S_Type || '',
value: f => f.labels.DstK8S_Type || '',
sort: (a, b, col) => compareStrings(col.value(a) as string, col.value(b) as string),
width: 10
},
Expand Down Expand Up @@ -640,7 +640,7 @@ export const getSrcDstColumns = (t: TFunction, withConcatenatedFields = true): C
getConcatenatedValue(
f.fields.SrcAddr,
f.fields.SrcPort || NaN,
f.fields.SrcK8S_Type,
f.labels.SrcK8S_Type,
f.labels.SrcK8S_Namespace,
f.fields.SrcK8S_Name
),
Expand Down Expand Up @@ -682,7 +682,7 @@ export const getSrcDstColumns = (t: TFunction, withConcatenatedFields = true): C
getConcatenatedValue(
f.fields.DstAddr,
f.fields.DstPort || NaN,
f.fields.DstK8S_Type,
f.labels.DstK8S_Type,
f.labels.DstK8S_Namespace,
f.fields.DstK8S_Name
),
Expand Down
Loading