Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[oracle] Missing total.bytes field in tablespace datastream #39787

Merged
merged 13 commits into from
Jun 13, 2024
Merged

[oracle] Missing total.bytes field in tablespace datastream #39787

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
5ec5122
fix total.bytes for tablespace
niraj-elastic Jun 3, 2024
4395915
update changelog
niraj-elastic Jun 3, 2024
6b89212
Merge branch 'main' into oracle_total_bytes_fix
kush-elastic Jun 3, 2024
2e7a998
fix CI errors
niraj-elastic Jun 4, 2024
683b39d
Merge branch 'oracle_total_bytes_fix' of github.com:niraj-elastic/bea…
niraj-elastic Jun 4, 2024
2ce18ac
fix CI
niraj-elastic Jun 4, 2024
0873951
fix CI issues
niraj-elastic Jun 4, 2024
296ed24
address review comments
niraj-elastic Jun 5, 2024
f99a0a0
address review comment
niraj-elastic Jun 5, 2024
c90771f
update changelog
niraj-elastic Jun 5, 2024
f0d0229
address review comment
niraj-elastic Jun 7, 2024
4b5160d
update visualization
niraj-elastic Jun 12, 2024
d092845
update visualization name
niraj-elastic Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627]
- Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710]
- Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664]
- Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787]

*Osquerybeat*

Expand Down
24 changes: 23 additions & 1 deletion ...beat/module/oracle/_meta/kibana/7/visualization/05acae50-9bf0-11e9-a61b-f742ed613c57.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,31 @@
"line_width": 1,
"metrics": [
{
"field": "oracle.tablespace.space.total.bytes",
"field": "oracle.tablespace.space.used.bytes",
"id": "61ca57f2-469d-11e7-af02-69e470af7417",
"type": "avg"
},
{
"field": "oracle.tablespace.space.free.bytes",
"id": "e04e8f40-24cd-4066-b12c-da0db0ff73d4",
"type": "avg"
},
{
"id": "2cf57800-8b54-41fa-a877-159b49699a50",
"script": "params.used_bytes + params.free_bytes",
"type": "math",
"variables": [
{
"field": "61ca57f2-469d-11e7-af02-69e470af7417",
"id": "631a44d5-d18a-4743-bea0-6f61930fd65f",
"name": "used_bytes"
},
{
"field": "e04e8f40-24cd-4066-b12c-da0db0ff73d4",
"id": "c255d24c-3a29-4879-b999-77af43d97c6b",
"name": "free_bytes"
}
]
}
],
"point_size": 1,
Expand Down
20 changes: 11 additions & 9 deletions x-pack/metricbeat/module/oracle/tablespace/data.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@ import (

// extract is the E of a ETL processing. Gets the data files, used/free space and temp free space data that is fetch
// by doing queries to Oracle
func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (out *extractedData, err error) {
out = &extractedData{}
func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMethods) (*extractedData, error) {
out := &extractedData{}
var err error

if out.dataFiles, err = extractor.dataFilesData(ctx); err != nil {
return nil, fmt.Errorf("error getting data_files: %w", err)
Expand All @@ -31,23 +32,23 @@ func (m *MetricSet) extract(ctx context.Context, extractor tablespaceExtractMeth
return nil, fmt.Errorf("error getting free space data: %w", err)
}

return
return out, nil
devamanv marked this conversation as resolved.
Show resolved Hide resolved
}

// transform is the T of an ETL (refer to the 'extract' method above if you need to see the origin). Transforms the data
// to create a Kibana/Elasticsearch friendly JSON. Data from Oracle is pretty fragmented by design so a lot of data
// was necessary. Data is organized by Tablespace entity (Tablespaces might contain one or more data files)
func (m *MetricSet) transform(in *extractedData) (out map[string]mapstr.M) {
out = make(map[string]mapstr.M, 0)
func (m *MetricSet) transform(in *extractedData) map[string]mapstr.M {
out := make(map[string]mapstr.M, 0)

for _, dataFile := range in.dataFiles {
m.addDataFileData(&dataFile, out)
for i := range in.dataFiles {
m.addDataFileData(&in.dataFiles[i], out)
}

m.addUsedAndFreeSpaceData(in.freeSpace, out)
m.addTempFreeSpaceData(in.tempFreeSpace, out)

return
return out
devamanv marked this conversation as resolved.
Show resolved Hide resolved
}

func (m *MetricSet) extractAndTransform(ctx context.Context) ([]mb.Event, error) {
Expand Down Expand Up @@ -78,7 +79,7 @@ func (m *MetricSet) addTempFreeSpaceData(tempFreeSpaces []tempFreeSpace, out map
name := val.(string)
if name == "TEMP" {
for _, tempFreeSpaceTable := range tempFreeSpaces {
oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TablespaceSize})
oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.TotalSpaceBytes})
oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.UsedSpaceBytes})
oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: tempFreeSpaceTable.FreeSpace})
}
Expand All @@ -101,6 +102,7 @@ func (m *MetricSet) addUsedAndFreeSpaceData(freeSpaces []usedAndFreeSpace, out m
if name == freeSpaceTable.TablespaceName {
oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.free.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalFreeBytes})
oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.used.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalUsedBytes})
oracle.SetSqlValueWithParentKey(m.Logger(), out, key, "space.total.bytes", &oracle.Int64Value{NullInt64: freeSpaceTable.TotalSpaceBytes})
}
}
}
Expand Down
12 changes: 6 additions & 6 deletions x-pack/metricbeat/module/oracle/tablespace/data_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ import (
"github.com/stretchr/testify/assert"
)

var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"used":{"bytes":9991}}}`,
var expectedResults = []string{`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux01.dbf","online_status":"ONLINE","size":{"bytes":9999990,"free":{"bytes":99999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":181,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux02.dbf","online_status":"ONLINE","size":{"bytes":9999991,"free":{"bytes":99999995},"max":{"bytes":9999995}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":182,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/sysaux03.dbf","online_status":"ONLINE","size":{"bytes":9999992,"free":{"bytes":99999996},"max":{"bytes":9999996}},"status":"AVAILABLE"},"name":"SYSAUX","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/system01.dbf","online_status":"ONLINE","size":{"bytes":999990,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"SYSTEM","space":{"free":{"bytes":9990},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/temp012017-03-02_07-54-38-075-AM.dbf","online_status":"ONLINE","size":{"bytes":999991,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"TEMP","space":{"free":{"bytes":99999},"total":{"bytes":99999},"used":{"bytes":99999}}}`,
`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"used":{"bytes":9991}}}`}
`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/undotbs01.dbf","online_status":"ONLINE","size":{"bytes":999992,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"UNDOTBS1","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`,
`{"data_file":{"id":18,"name":"/u02/app/oracle/oradata/ORCLCDB/orclpdb1/users01.dbf","online_status":"ONLINE","size":{"bytes":999993,"free":{"bytes":9999994},"max":{"bytes":9999994}},"status":"AVAILABLE"},"name":"USERS","space":{"free":{"bytes":9999},"total":{"bytes":99999},"used":{"bytes":9991}}}`}

var notExpectedEvents = []string{`{}`, `{"foo":"bar"}`}

Expand Down
10 changes: 5 additions & 5 deletions x-pack/metricbeat/module/oracle/tablespace/mocks_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,16 @@ func (h happyDataFiles) dataFilesData(_ context.Context) ([]dataFile, error) {
type happyTempFreeSpaceData struct{}

func (happyTempFreeSpaceData) tempFreeSpaceData(_ context.Context) ([]tempFreeSpace, error) {
return []tempFreeSpace{{TablespaceName: "TEMP", TablespaceSize: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil
return []tempFreeSpace{{TablespaceName: "TEMP", TotalSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, UsedSpaceBytes: sql.NullInt64{Valid: true, Int64: 99999}, FreeSpace: sql.NullInt64{Int64: 99999, Valid: true}}}, nil
}

type happyFreeSpaceData struct{}

func (happyFreeSpaceData) usedAndFreeSpaceData(_ context.Context) ([]usedAndFreeSpace, error) {
return []usedAndFreeSpace{
{TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
{TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
{TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
{TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}},
{TablespaceName: "SYSTEM", TotalFreeBytes: sql.NullInt64{Int64: 9990, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
{TablespaceName: "SYSAUX", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
{TablespaceName: "UNDOTBS1", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
{TablespaceName: "USERS", TotalFreeBytes: sql.NullInt64{Int64: 9999, Valid: true}, TotalUsedBytes: sql.NullInt64{Int64: 9991, Valid: true}, TotalSpaceBytes: sql.NullInt64{Int64: 99999, Valid: true}},
}, nil
}
21 changes: 7 additions & 14 deletions x-pack/metricbeat/module/oracle/tablespace/temp_free_space.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,31 +11,24 @@ import (
)

type tempFreeSpace struct {
TablespaceName string
TablespaceSize sql.NullInt64
UsedSpaceBytes sql.NullInt64
FreeSpace sql.NullInt64
}

func (d *tempFreeSpace) hash() string {
return d.TablespaceName
}

func (d *tempFreeSpace) eventKey() string {
return d.TablespaceName
TablespaceName string
TotalSpaceBytes sql.NullInt64
UsedSpaceBytes sql.NullInt64
FreeSpace sql.NullInt64
}

func (e *tablespaceExtractor) tempFreeSpaceData(ctx context.Context) ([]tempFreeSpace, error) {
rows, err := e.db.QueryContext(ctx, "SELECT TABLESPACE_NAME, TABLESPACE_SIZE, ALLOCATED_SPACE, FREE_SPACE FROM DBA_TEMP_FREE_SPACE")
rows, err := e.db.QueryContext(ctx, `WITH sums AS ( SELECT (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM FROM dual ) SELECT t.TABLESPACE_NAME, s.TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE FROM DBA_TEMP_FREE_SPACE t, sums s `)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you consider something simple as below

SELECT 
    t.TABLESPACE_NAME, 
    (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM, 
    t.ALLOCATED_SPACE, 
    t.FREE_SPACE 
FROM 
    DBA_TEMP_FREE_SPACE t;

Copy link
Member

@shmsr shmsr Jun 6, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Talking about Niraj's query:

See this. (You are using cross join with CTE and DBA_TEMP_FREE_SPACE).

Now that I read more and more on this I keep on finding something new.

In the query below, the scalar subquery is used to get the TOTAL_SUM from sums CTE compared to your query where CTE is cross-joined with DBA_TEMP_FREE_SPACE. From what I read in most places, scalar subquery seems to be generally a better choice in terms of perf and readability. For example, As the same scalar subquery is running for each row, Oracle might as well cache it and keep using it. In the case of cross joins, it takes up a lot of memory to create a temp table to do the join (unless the DB optimizes it).

Find more here: See: https://stackoverflow.com/a/55192080/5821408 and https://asktom.oracle.com/Misc/oramag/on-caching-and-evangelizing-sql.html

Try this query?

WITH sums AS (
  SELECT (SELECT SUM(BYTES) FROM DBA_DATA_FILES) + (SELECT SUM(BYTES) FROM DBA_TEMP_FILES) AS TOTAL_SUM
  FROM dual
)
SELECT t.TABLESPACE_NAME, (SELECT TOTAL_SUM FROM sums) AS TOTAL_SUM, t.ALLOCATED_SPACE, t.FREE_SPACE
FROM DBA_TEMP_FREE_SPACE t;

This query will calculate the total sum separately and display it as a single value for each row. No cross joins.

if err != nil {
return nil, fmt.Errorf("error executing query: %w", err)
}
defer rows.Close()

results := make([]tempFreeSpace, 0)

for rows.Next() {
dest := tempFreeSpace{}
if err = rows.Scan(&dest.TablespaceName, &dest.TablespaceSize, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil {
if err = rows.Scan(&dest.TablespaceName, &dest.TotalSpaceBytes, &dest.UsedSpaceBytes, &dest.FreeSpace); err != nil {
return nil, err
}
results = append(results, dest)
Expand Down
20 changes: 7 additions & 13 deletions x-pack/metricbeat/module/oracle/tablespace/used_and_free_space.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,30 +11,24 @@ import (
)

type usedAndFreeSpace struct {
TablespaceName string
TotalFreeBytes sql.NullInt64
TotalUsedBytes sql.NullInt64
}

func (d *usedAndFreeSpace) hash() string {
return d.TablespaceName
}

func (d *usedAndFreeSpace) eventKey() string {
return d.TablespaceName
TablespaceName string
TotalSpaceBytes sql.NullInt64
TotalFreeBytes sql.NullInt64
TotalUsedBytes sql.NullInt64
}

func (e *tablespaceExtractor) usedAndFreeSpaceData(ctx context.Context) ([]usedAndFreeSpace, error) {
rows, err := e.db.QueryContext(ctx, "SELECT b.tablespace_name, tbs_size used, a.free_space free FROM (SELECT tablespace_name, sum(bytes) AS free_space FROM dba_free_space GROUP BY tablespace_name) a, (SELECT tablespace_name, sum(bytes) AS tbs_size FROM dba_data_files GROUP BY tablespace_name) b WHERE a.tablespace_name(+)=b.tablespace_name")
rows, err := e.db.QueryContext(ctx, `SELECT b.tablespace_name, (b.tbs_size - a.free_space) used, a.free_space free, (SELECT SUM(bytes) FROM DBA_DATA_FILES) + (SELECT SUM(bytes) FROM DBA_TEMP_FILES) AS total_sum FROM (SELECT tablespace_name, SUM(bytes) AS free_space FROM DBA_FREE_SPACE GROUP BY tablespace_name) a RIGHT JOIN (SELECT tablespace_name, SUM(bytes) AS tbs_size FROM DBA_DATA_FILES GROUP BY tablespace_name) b ON a.tablespace_name = b.tablespace_name`)
shmsr marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, fmt.Errorf("error executing query: %w", err)
}
defer rows.Close()

results := make([]usedAndFreeSpace, 0)

for rows.Next() {
dest := usedAndFreeSpace{}
if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes); err != nil {
if err = rows.Scan(&dest.TablespaceName, &dest.TotalUsedBytes, &dest.TotalFreeBytes, &dest.TotalSpaceBytes); err != nil {
return nil, err
}
results = append(results, dest)
Expand Down
Loading