Skip to content
This repository has been archived by the owner on Apr 2, 2024. It is now read-only.

Commit

Permalink
Deprecate testing for TimescaleDB 1.x
Browse files Browse the repository at this point in the history
  • Loading branch information
@cevian
cevian committed Mar 9, 2022
1 parent 30b2139 commit bfa90a1
Show file tree
Hide file tree
Showing 21 changed files with 61 additions and 480 deletions.
16 changes: 7 additions & 9 deletions .github/workflows/go-scheduled.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,13 +77,12 @@ jobs:
strategy:
matrix:
test-setups:
- {name: "Multinode", ext: true, tsdb: true, tsdb2: true, multi: true, pg: 13, nightly: false}
- {name: "W/O Promscale Extension", ext: false, tsdb: true, tsdb2: true, multi: false, pg: 13, nightly: false}
- {name: "Plain Postgres", ext: false, tsdb: false, tsdb2: false, multi: false, pg: 13, nightly: false}
- {name: "Timescaledb 1.x (pg 12)", ext: true, tsdb: true, tsdb2: false, multi: false, pg: 12, nightly: false}
- {name: "Plain Postgres (12)", ext: false, tsdb: false, tsdb2: false, multi: false, pg: 12, nightly: false}
- {name: "TimescaleDB Nightly (PG-13)", ext: false, tsdb: true, tsdb2: true, multi: false, pg: 13, nightly: true}
- {name: "TimescaleDB-Multinode Nightly (PG-13)", ext: false, tsdb: true, tsdb2: true, multi: true, pg: 13, nightly: true}
- {name: "Multinode", ext: true, tsdb: true, multi: true, pg: 13, nightly: false}
- {name: "W/O Promscale Extension", ext: false, tsdb: true, multi: false, pg: 13, nightly: false}
- {name: "Plain Postgres", ext: false, tsdb: false, multi: false, pg: 13, nightly: false}
- {name: "Plain Postgres (12)", ext: false, tsdb: false, multi: false, pg: 12, nightly: false}
- {name: "TimescaleDB Nightly (PG-13)", ext: false, tsdb: true, multi: false, pg: 13, nightly: true}
- {name: "TimescaleDB-Multinode Nightly (PG-13)", ext: false, tsdb: true, multi: true, pg: 13, nightly: true}
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
Expand Down Expand Up @@ -112,10 +111,9 @@ jobs:
PG: ${{ matrix.test-setups.pg }}
EXT: ${{ matrix.test-setups.ext }}
TSDB: ${{ matrix.test-setups.tsdb }}
TSDB2: ${{ matrix.test-setups.tsdb2 }}
MULTI: ${{ matrix.test-setups.multi }}
NIGHTLY: ${{ matrix.test-setups.nightly }}
run: go test ./pkg/tests/end_to_end_tests/ -use-extension=$EXT -use-timescaledb=$TSDB -use-timescale2=$TSDB2 -use-multinode=$MULTI -use-timescaledb-nightly=$NIGHTLY -postgres-version-major=$PG
run: go test ./pkg/tests/end_to_end_tests/ -use-extension=$EXT -use-timescaledb=$TSDB -use-multinode=$MULTI -use-timescaledb-nightly=$NIGHTLY -postgres-version-major=$PG

test-helm-chart:
name: Test helm-chart template generation
Expand Down
18 changes: 8 additions & 10 deletions .github/workflows/go.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,14 +91,13 @@ jobs:
strategy:
matrix:
test-setups:
- {name: "Timescaledb 1.x (pg 12)", shortname: "tsdb1x-pg-12", ext: true, tsdb: true, tsdb2: false, tsdboss: false, multi: false, pg: 12}
- {name: "Plain Postgres (12)", shortname: "plain-pg-12", ext: false, tsdb: false, tsdb2: false, tsdboss: false, multi: false, pg: 12}
- {name: "Plain Postgres (13)", shortname: "plain-pg-13", ext: false, tsdb: false, tsdb2: false, tsdboss: false, multi: false, pg: 13}
- {name: "Singlenode", shortname: "singlenode", ext: true, tsdb: true, tsdb2: true, tsdboss: false, multi: false, pg: 14}
- {name: "Multinode", shortname: "multinode", ext: true, tsdb: true, tsdb2: true, tsdboss: false, multi: true, pg: 14}
- {name: "W/O Promscale Extension", shortname: "wo-prom-ext", ext: false, tsdb: true, tsdb2: true, tsdboss: false, multi: false, pg: 14}
- {name: "Plain Postgres", shortname: "plain-pg", ext: false, tsdb: false, tsdb2: false, tsdboss: false, multi: false, pg: 14}
- {name: "TimescaleDB-OSS", shortname: "tsdb-oss", ext: false, tsdb: true, tsdb2: true, tsdboss: true, multi: false, pg: 14}
- {name: "Plain Postgres (12)", shortname: "plain-pg-12", ext: false, tsdb: false, tsdboss: false, multi: false, pg: 12}
- {name: "Plain Postgres (13)", shortname: "plain-pg-13", ext: false, tsdb: false, tsdboss: false, multi: false, pg: 13}
- {name: "Plain Postgres", shortname: "plain-pg", ext: false, tsdb: false, tsdboss: false, multi: false, pg: 14}
- {name: "Singlenode", shortname: "singlenode", ext: true, tsdb: true, tsdboss: false, multi: false, pg: 14}
- {name: "Multinode", shortname: "multinode", ext: true, tsdb: true, tsdboss: false, multi: true, pg: 14}
- {name: "W/O Promscale Extension", shortname: "wo-prom-ext", ext: false, tsdb: true, tsdboss: false, multi: false, pg: 14}
- {name: "TimescaleDB-OSS", shortname: "tsdb-oss", ext: false, tsdb: true, tsdboss: true, multi: false, pg: 14}

steps:
- name: Check out code into the Go module directory
Expand Down Expand Up @@ -126,11 +125,10 @@ jobs:
PG: ${{ matrix.test-setups.pg }}
EXT: ${{ matrix.test-setups.ext }}
TSDB: ${{ matrix.test-setups.tsdb }}
TSDB2: ${{ matrix.test-setups.tsdb2 }}
MULTI: ${{ matrix.test-setups.multi }}
TSDBOSS: ${{ matrix.test-setups.tsdboss }}
SHORTNAME: ${{ matrix.test-setups.shortname }}
run: go test -race -timeout=30m ./pkg/tests/end_to_end_tests/ -use-extension=$EXT -use-timescaledb=$TSDB -use-timescale2=$TSDB2 -use-timescaledb-oss=$TSDBOSS -use-multinode=$MULTI -postgres-version-major=$PG | tee $SHORTNAME-test-run.log 2>&1
run: go test -race -timeout=30m ./pkg/tests/end_to_end_tests/ -use-extension=$EXT -use-timescaledb=$TSDB -use-timescaledb-oss=$TSDBOSS -use-multinode=$MULTI -postgres-version-major=$PG | tee $SHORTNAME-test-run.log 2>&1

- name: 'Upload Log Artifact'
if: ${{ always() }}
Expand Down
45 changes: 12 additions & 33 deletions pkg/internal/testhelpers/postgres_container.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ type ExtensionState int
const (
timescaleBit = 1 << iota
promscaleBit = 1 << iota
timescale2Bit = 1 << iota
multinodeBit = 1 << iota
postgres12Bit = 1 << iota
postgres13Bit = 1 << iota
Expand All @@ -64,16 +63,14 @@ const (

const (
VanillaPostgres ExtensionState = 0
Timescale1 ExtensionState = timescaleBit
Timescale1AndPromscale ExtensionState = timescaleBit | promscaleBit
Timescale2 ExtensionState = timescaleBit | timescale2Bit
Timescale2AndPromscale ExtensionState = timescaleBit | timescale2Bit | promscaleBit
Multinode ExtensionState = timescaleBit | timescale2Bit | multinodeBit
MultinodeAndPromscale ExtensionState = timescaleBit | timescale2Bit | multinodeBit | promscaleBit
TimescaleOSS ExtensionState = timescaleBit | timescale2Bit | timescaleOSSBit

TimescaleNightly ExtensionState = timescaleBit | timescale2Bit | timescaleNightlyBit
TimescaleNightlyMultinode ExtensionState = timescaleBit | timescale2Bit | multinodeBit | timescaleNightlyBit
Timescale2 ExtensionState = timescaleBit
Timescale2AndPromscale ExtensionState = timescaleBit | promscaleBit
Multinode ExtensionState = timescaleBit | multinodeBit
MultinodeAndPromscale ExtensionState = timescaleBit | multinodeBit | promscaleBit
TimescaleOSS ExtensionState = timescaleBit | timescaleOSSBit

TimescaleNightly ExtensionState = timescaleBit | timescaleNightlyBit
TimescaleNightlyMultinode ExtensionState = timescaleBit | multinodeBit | timescaleNightlyBit
)

func (e *ExtensionState) UsePromscale() {
Expand All @@ -84,20 +81,16 @@ func (e *ExtensionState) UseTimescaleDB() {
*e |= timescaleBit
}

func (e *ExtensionState) UseTimescaleDB2() {
*e |= timescaleBit | timescale2Bit
}

func (e *ExtensionState) UseTimescaleNightly() {
*e |= timescaleBit | timescale2Bit | timescaleNightlyBit
*e |= timescaleBit | timescaleNightlyBit
}

func (e *ExtensionState) UseTimescaleNightlyMultinode() {
*e |= timescaleBit | timescale2Bit | multinodeBit | timescaleNightlyBit
*e |= timescaleBit | multinodeBit | timescaleNightlyBit
}

func (e *ExtensionState) UseMultinode() {
*e |= timescaleBit | timescale2Bit | multinodeBit
*e |= timescaleBit | multinodeBit
}

func (e *ExtensionState) UsePG12() {
Expand All @@ -109,17 +102,13 @@ func (e *ExtensionState) UsePG13() {
}

func (e *ExtensionState) UseTimescaleDBOSS() {
*e |= timescaleBit | timescale2Bit | timescaleOSSBit
*e |= timescaleBit | timescaleOSSBit
}

func (e ExtensionState) UsesTimescaleDB() bool {
return (e & timescaleBit) != 0
}

func (e ExtensionState) UsesTimescale2() bool {
return (e & timescale2Bit) != 0
}

func (e ExtensionState) UsesTimescaleNightly() bool {
return (e & timescaleNightlyBit) != 0
}
Expand Down Expand Up @@ -160,16 +149,6 @@ func (e ExtensionState) GetDockerImageName() (string, error) {
PGTag := "pg" + PGMajor

switch e &^ postgres12Bit &^ postgres13Bit {
case Timescale1:
if PGMajor != "12" {
return "", fmt.Errorf("timescaledb 1.x requires pg12")
}
image = "timescale/timescaledb:1.7.4-pg12"
case Timescale1AndPromscale:
if PGMajor != "12" {
return "", fmt.Errorf("timescaledb 1.x requires pg12")
}
image = LatestDBWithPromscaleImageBase + ":latest-ts1-pg12"
case Timescale2, Multinode:
image = "timescale/timescaledb:latest-" + PGTag
case Timescale2AndPromscale, MultinodeAndPromscale:
Expand Down
47 changes: 8 additions & 39 deletions pkg/tests/end_to_end_tests/continuous_agg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ package end_to_end_tests
import (
"context"
"fmt"
"github.com/timescale/promscale/pkg/pgmodel/common/schema"
"testing"
"time"

"github.com/timescale/promscale/pkg/pgmodel/common/schema"

"github.com/jackc/pgx/v4/pgxpool"
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/labels"
Expand Down Expand Up @@ -186,29 +187,13 @@ func TestContinuousAggDownsampling(t *testing.T) {
if _, err := db.Exec(context.Background(), "CREATE SCHEMA cagg_schema"); err != nil {
t.Fatalf("unexpected error while creating view schema: %s", err)
}
if *useTimescale2 {
if _, err := db.Exec(context.Background(),
`CREATE MATERIALIZED VIEW cagg_schema.cagg( time, series_id, value, max, min, avg)
if _, err := db.Exec(context.Background(),
`CREATE MATERIALIZED VIEW cagg_schema.cagg( time, series_id, value, max, min, avg)
WITH (timescaledb.continuous) AS
SELECT time_bucket('1hour', time), series_id, max(value) as value, max(value) as max, min(value) as min, avg(value) as avg
FROM prom_data.metric_2
GROUP BY time_bucket('1hour', time), series_id`); err != nil {
t.Fatalf("unexpected error while creating metric view: %s", err)
}
} else {
// Using TimescaleDB 1.x
if _, err := db.Exec(context.Background(),
`CREATE VIEW cagg_schema.cagg( time, series_id, value, max, min, avg)
WITH (timescaledb.continuous, timescaledb.ignore_invalidation_older_than = '1 min', timescaledb.refresh_lag = '-2h') AS
SELECT time_bucket('1hour', time), series_id, max(value) as value, max(value) as max, min(value) as min, avg(value) as avg
FROM prom_data.metric_2
GROUP BY time_bucket('1hour', time), series_id`); err != nil {
t.Fatalf("unexpected error while creating metric view: %s", err)
}
if _, err := db.Exec(context.Background(),
`REFRESH MATERIALIZED VIEW cagg_schema.cagg`); err != nil {
t.Fatalf("unexpected error while creating metric view: %s", err)
}
t.Fatalf("unexpected error while creating metric view: %s", err)
}
if _, err := db.Exec(context.Background(), "SELECT prom_api.register_metric_view('cagg_schema', 'cagg')"); err != nil {
t.Fatalf("unexpected error while registering metric view: %s", err)
Expand Down Expand Up @@ -391,26 +376,13 @@ func TestContinuousAggDataRetention(t *testing.T) {
_, err = db.Exec(context.Background(), "CREATE SCHEMA cagg_schema")
require.NoError(t, err)

if *useTimescale2 {
_, err = db.Exec(context.Background(),
`CREATE MATERIALIZED VIEW cagg_schema.cagg( time, series_id, value, max, min, avg)
_, err = db.Exec(context.Background(),
`CREATE MATERIALIZED VIEW cagg_schema.cagg( time, series_id, value, max, min, avg)
WITH (timescaledb.continuous) AS
SELECT time_bucket('1hour', time), series_id, max(value) as value, max(value) as max, min(value) as min, avg(value) as avg
FROM prom_data.test
GROUP BY time_bucket('1hour', time), series_id`)
require.NoError(t, err)
} else {
// Using TimescaleDB 1.x
_, err = db.Exec(context.Background(),
`CREATE VIEW cagg_schema.cagg( time, series_id, value, max, min, avg)
WITH (timescaledb.continuous, timescaledb.max_interval_per_job = '1000 weeks', timescaledb.ignore_invalidation_older_than = '1 min') AS
SELECT time_bucket('1hour', time), series_id, max(value) as value, max(value) as max, min(value) as min, avg(value) as avg
FROM prom_data.test
GROUP BY time_bucket('1hour', time), series_id`)
require.NoError(t, err)
_, err = db.Exec(context.Background(), `REFRESH MATERIALIZED VIEW cagg_schema.cagg`)
require.NoError(t, err)
}
require.NoError(t, err)
_, err = db.Exec(context.Background(), "SELECT prom_api.register_metric_view('cagg_schema', 'cagg')")
require.NoError(t, err)

Expand Down Expand Up @@ -449,9 +421,6 @@ func TestContinuousAgg2StepAgg(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
if !*useTimescale2 {
t.Skip("2-step continuous aggregates need TimescaleDB 2.x support")
}
if !*useExtension {
t.Skip("2-step continuous aggregates need TimescaleDB 2.x HA image")
}
Expand Down
53 changes: 3 additions & 50 deletions pkg/tests/end_to_end_tests/create_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (

"github.com/jackc/pgconn"
"github.com/jackc/pgerrcode"
pgx "github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
_ "github.com/jackc/pgx/v4/stdlib"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -755,15 +754,9 @@ func TestInsertCompressed(t *testing.T) {
t.Fatal(err)
}
var nextStartAfter time.Time
var statsQuery string
if *useTimescale2 {
statsQuery = "SELECT delay_compression_until FROM _prom_catalog.metric " +
"WHERE metric_name = $1::text"
err = db.QueryRow(context.Background(), statsQuery, "Test").Scan(&nextStartAfter)
} else {
statsQuery = "SELECT next_start FROM timescaledb_information.policy_stats WHERE hypertable = $1::text::regclass"
err = db.QueryRow(context.Background(), statsQuery, pgx.Identifier{"prom_data", tableName}.Sanitize()).Scan(&nextStartAfter)
}
statsQuery := "SELECT delay_compression_until FROM _prom_catalog.metric " +
"WHERE metric_name = $1::text"
err = db.QueryRow(context.Background(), statsQuery, "Test").Scan(&nextStartAfter)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -901,22 +894,6 @@ func TestCompressionSetting(t *testing.T) {
withDB(t, *testDatabase, func(dbOwner *pgxpool.Pool, t testing.TB) {
db := testhelpers.PgxPoolWithRole(t, *testDatabase, "prom_admin")
defer db.Close()
checkJobs := func(t testing.TB, jobs_expected int) {
countQuery := ""
jobs := make([]string, 0)
if *useTimescale2 {
//compression does not effect #jobs in timescaledb2
return
}
countQuery = "SELECT array_agg((s.*)::text) FROM _timescaledb_config.bgw_job s"
err := db.QueryRow(context.Background(), countQuery).Scan(&jobs)
if err != nil {
t.Fatal(err)
}
if len(jobs) != jobs_expected {
t.Errorf("unexpected jobs, expected 1 got %v", jobs)
}
}

var compressionEnabled bool
err := db.QueryRow(context.Background(), "SELECT _prom_catalog.get_default_compression_setting()").Scan(&compressionEnabled)
Expand All @@ -928,8 +905,6 @@ func TestCompressionSetting(t *testing.T) {
t.Error("compression should be enabled by default, was not")
}

checkJobs(t, 1)

_, err = db.Exec(context.Background(), "SELECT prom_api.set_default_compression_setting(false)")
if err != nil {
t.Fatal(err)
Expand All @@ -943,8 +918,6 @@ func TestCompressionSetting(t *testing.T) {

}

checkJobs(t, 1)

ts := []prompb.TimeSeries{
{
Labels: []prompb.Label{
Expand Down Expand Up @@ -979,8 +952,6 @@ func TestCompressionSetting(t *testing.T) {
t.Error("metric compression should be disabled as per default, was not")
}

checkJobs(t, 1)

_, err = db.Exec(context.Background(), "SELECT prom_api.set_metric_compression_setting('Test', true)")
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -1010,8 +981,6 @@ func TestCompressionSetting(t *testing.T) {
t.Fatal("metric compression should be enabled manually, was not")
}

checkJobs(t, 2)

if *useMultinode {
//TODO turning compression off in multinode is broken upstream.
return
Expand All @@ -1030,7 +999,6 @@ func TestCompressionSetting(t *testing.T) {
if compressionEnabled {
t.Error("metric compression should be disabled as per default, was not")
}
checkJobs(t, 1)
})
}

Expand All @@ -1041,9 +1009,6 @@ func TestCustomCompressionJob(t *testing.T) {
if !*useTimescaleDB {
t.Skip("compression meaningless without TimescaleDB")
}
if !*useTimescale2 {
t.Skip("test meaningless without Timescale 2")
}
if *useTimescaleOSS {
t.Skip("compression not applicable in TimescaleDB-OSS")
}
Expand Down Expand Up @@ -1256,9 +1221,6 @@ func TestExecuteMaintenanceCompressionJob(t *testing.T) {
if !*useTimescaleDB {
t.Skip("compression meaningless without TimescaleDB")
}
if !*useTimescale2 {
t.Skip("test meaningless without Timescale 2")
}
if *useTimescaleOSS {
t.Skip("compression not applicable in TimescaleDB-OSS")
}
Expand Down Expand Up @@ -1424,9 +1386,6 @@ func TestExecuteCompressionMetricsLocked(t *testing.T) {
if !*useTimescaleDB {
t.Skip("compression meaningless without TimescaleDB")
}
if !*useTimescale2 {
t.Skip("test meaningless without Timescale 2")
}
if *useTimescaleOSS {
t.Skip("compression not applicable in TimescaleDB-OSS")
}
Expand Down Expand Up @@ -1558,9 +1517,6 @@ func TestConfigMaintenanceJobs(t *testing.T) {
if !*useTimescaleDB {
t.Skip("jobs meaningless without TimescaleDB")
}
if !*useTimescale2 {
t.Skip("test meaningless without Timescale 2")
}
if *useTimescaleOSS {
t.Skip("add_job() is not available in TimescaleDB OSS, hence this test is not applicable")
}
Expand Down Expand Up @@ -1654,9 +1610,6 @@ func TestExecuteMaintJob(t *testing.T) {
if !*useTimescaleDB {
t.Skip("jobs meaningless without TimescaleDB")
}
if !*useTimescale2 {
t.Skip("test meaningless without Timescale 2")
}
withDB(t, *testDatabase, func(dbOwner *pgxpool.Pool, t testing.TB) {
dbSuper, err := pgxpool.Connect(context.Background(), testhelpers.PgConnectURL(*testDatabase, testhelpers.Superuser))
if err != nil {
Expand Down
Loading

0 comments on commit bfa90a1

Please sign in to comment.