stats: add a histogram version number

This commit adds a histogram version number to the HistogramData proto. This will allow us to identify what logic was used to construct a particular histogram and possibly debug future issues. Release note: None Release justification: Low risk, high benefit change to existing functionality.
cockroachdb · Sep 9, 2021 · 78191f2 · 78191f2
1 parent 7792137
commit 78191f2
Show file tree

Hide file tree

Showing 29 changed files with 16,131 additions and 15,875 deletions.
diff --git a/pkg/sql/logictest/testdata/logic_test/distsql_stats b/pkg/sql/logictest/testdata/logic_test/distsql_stats
@@ -1093,6 +1093,7 @@ FROM [SHOW STATISTICS USING JSON FOR TABLE all_null]
             }
         ],
         "histo_col_type": "INT8",
+        "histo_version": 1,
         "name": "s",
         "null_count": 0,
         "row_count": 1
@@ -1141,6 +1142,7 @@ FROM [SHOW STATISTICS USING JSON FOR TABLE greeting_stats]
             }
         ],
         "histo_col_type": "public.greeting",
+        "histo_version": 1,
         "name": "s",
         "null_count": 0,
         "row_count": 1

diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpcc b/pkg/sql/opt/memo/testdata/stats_quality/tpcc
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q01 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q01
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q02 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q02
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q03 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q03
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q04 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q04
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q05 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q05
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q06 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q06
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q07 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q07
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q08 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q08
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q09 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q09
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q10 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q10
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q12 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q12
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q13 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q13
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q14 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q14
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q15 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q15
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q16 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q16
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q17 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q17
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q18 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q18
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q19 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q19
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q20 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q20
diff --git a/pkg/sql/opt/memo/testdata/stats_quality/tpch/q22 b/pkg/sql/opt/memo/testdata/stats_quality/tpch/q22
@@ -94,12 +94,12 @@ sort
       │    │    │    │    ├── save-table-name: q22_scan_6
       │    │    │    │    ├── columns: c_custkey:1(int!null) c_phone:5(char!null) c_acctbal:6(float!null)
       │    │    │    │    ├── stats: [rows=150000, distinct(1)=148813, null(1)=0, distinct(5)=150000, null(5)=0, distinct(6)=140426, null(6)=0]
-      │    │    │    │    │   histogram(1)=  0  6   772   6   839   6    793   6    666   6    657   6    737   6    787   6    755   6    621   6    752   6    707   6    714   6    756   6    727    6    859    6    729    6    664    6    754    6    751    6    650    6    797    6    718    6    729    6    729    6    807    6    830    6    793    6    673    6    733    6    762    6    690    6    706    6    889    6    728    6    768    6    743    6    734    6    754    6    744    6    753    6    805    6    679    6    757    6    706    6    802    6    687    6    787    6    778    6    780    6    829    6    748    6    718    6    838    6    746    6    700    6    783    6    792    6    698    6    770    6    771    6    713    6    692    6    803    6    680    6    684    6    677    6    693    6    700    6    766    6    749    6    664    6    783    6    699    6    787    6    698    6    702    6    726    6    704    6    662    6    754    6    832    6    646    6    748    6    760    6    689    6    725    6    690    6    758    6    719    6    691    6    650    6    741    6    725    6    703    6    741    6    773    6    773    6    662    6    717    6    748    6    956    6    761    6    726    6    649    6    712    6    768    6    695    6    717    6    736    6    721    6    748    6    690    6    728    6    790    6    633    6    778    6    935    6    664    6    688    6    683    6    693    6    741    6    729    6    731    6    798    6    689    6    677    6    774    6    817    6    821    6    773    6    724    6    714    6    687    6    716    6     857    6     804    6     822    6     875    6     645    6     787    6     718    6     801    6     762    6     757    6     747    6     635    6     688    6     822    6     675    6     726    6     809    6     784    6     681    6     777    6     704    6     901    6     805    6     818    6     656    6     731    6     719    6     721    6     833    6     798    6     779    6     814    6     745    6     773    6     783    6     790    6     876    6     746    6     728    6     774    6     802    6     709    6     694    6     805    6     771    6     731    6     702    6     683    6     835    6     718    6     716    6     732    6     756    6     795    6     697    6     736    6     817    6     825    6     797    6     808    6     856    6     715    6     797    6     715    6
-      │    │    │    │    │                <--- 24 ----- 831 ----- 1757 ----- 2603 ----- 3203 ----- 3785 ----- 4528 ----- 5363 ----- 6140 ----- 6637 ----- 7408 ----- 8093 ----- 8791 ----- 9570 ----- 10294 ----- 11255 ----- 11983 ----- 12580 ----- 13354 ----- 14124 ----- 14691 ----- 15543 ----- 16249 ----- 16977 ----- 17705 ----- 18576 ----- 19486 ----- 20332 ----- 20948 ----- 21684 ----- 22474 ----- 23126 ----- 23810 ----- 24820 ----- 25546 ----- 26346 ----- 27101 ----- 27839 ----- 28613 ----- 29370 ----- 30143 ----- 31009 ----- 31638 ----- 32418 ----- 33101 ----- 33962 ----- 34607 ----- 35441 ----- 36260 ----- 37083 ----- 37992 ----- 38756 ----- 39463 ----- 40388 ----- 41148 ----- 41819 ----- 42646 ----- 43490 ----- 44157 ----- 44961 ----- 45767 ----- 46464 ----- 47120 ----- 47983 ----- 48614 ----- 49253 ----- 49877 ----- 50534 ----- 51206 ----- 52003 ----- 52768 ----- 53364 ----- 54191 ----- 54861 ----- 55696 ----- 56364 ----- 57039 ----- 57761 ----- 58441 ----- 59034 ----- 59809 ----- 60723 ----- 61281 ----- 62045 ----- 62830 ----- 63479 ----- 64200 ----- 64852 ----- 65634 ----- 66342 ----- 66995 ----- 67560 ----- 68311 ----- 69032 ----- 69710 ----- 70461 ----- 71270 ----- 72079 ----- 72672 ----- 73377 ----- 74140 ----- 75260 ----- 76048 ----- 76770 ----- 77334 ----- 78029 ----- 78829 ----- 79490 ----- 80195 ----- 80937 ----- 81649 ----- 82413 ----- 83064 ----- 83790 ----- 84631 ----- 85157 ----- 85975 ----- 87062 ----- 87659 ----- 88305 ----- 88942 ----- 89599 ----- 90349 ----- 91076 ----- 91807 ----- 92661 ----- 93310 ----- 93934 ----- 94746 ----- 95634 ----- 96529 ----- 97339 ----- 98058 ----- 98757 ----- 99401 ----- 100103 ----- 101060 ----- 101925 ----- 102821 ----- 103809 ----- 104364 ----- 105198 ----- 105905 ----- 106764 ----- 107554 ----- 108335 ----- 109096 ----- 109627 ----- 110274 ----- 111171 ----- 111792 ----- 112499 ----- 113362 ----- 114180 ----- 114796 ----- 115600 ----- 116264 ----- 117284 ----- 118139 ----- 119017 ----- 119578 ----- 120295 ----- 120990 ----- 121688 ----- 122593 ----- 123435 ----- 124244 ----- 125116 ----- 125861 ----- 126659 ----- 127474 ----- 128302 ----- 129281 ----- 130028 ----- 130739 ----- 131538 ----- 132388 ----- 133062 ----- 133705 ----- 134560 ----- 135353 ----- 136070 ----- 136729 ----- 137349 ----- 138257 ----- 138949 ----- 139638 ----- 140358 ----- 141124 ----- 141961 ----- 142611 ----- 143338 ----- 144215 ----- 145106 ----- 145947 ----- 146808 ----- 147752 ----- 148439 ----- 149280 ----- 149967
+      │    │    │    │    │   histogram(1)=  0  5   769   5   765   5    732   5    744   5    731   5    754   5    772   5    757   5    713   5    741   5    808   5    744   5    739   5    687    5    820    5    761    5    782    5    632    5    711    5    692    5    648    5    770    5    765    5    702    5    751    5    807    5    794    5    735    5    807    5    719    5    773    5    781    5    684    5    748    5    682    5    703    5    794    5    718    5    807    5    674    5    747    5    677    5    813    5    666    5    766    5    822    5    703    5    676    5    765    5    693    5    723    5    780    5    793    5    770    5    696    5    775    5    764    5    884    5    696    5    688    5    637    5    789    5    702    5    732    5    697    5    769    5    739    5    744    5    861    5    791    5    726    5    793    5    730    5    763    5    789    5    797    5    775    5    862    5    780    5    746    5    783    5    743    5    822    5    806    5    775    5    727    5    724    5    799    5    707    5    757    5    614    5    747    5    704    5    740    5    749    5    735    5    741    5    807    5    827    5    816    5    702    5    699    5    803    5    793    5    672    5    831    5    694    5    746    5    731    5    686    5    685    5    695    5    828    5    756    5    722    5    749    5    790    5    758    5    750    5    782    5    733    5    778    5    762    5    758    5    731    5    778    5    663    5    696    5    684    5    796    5    770    5    656    5    690    5     747    5     782    5     785    5     751    5     697    5     663    5     766    5     695    5     866    5     813    5     765    5     901    5     747    5     683    5     706    5     689    5     734    5     715    5     752    5     855    5     771    5     717    5     794    5     760    5     827    5     747    5     757    5     767    5     726    5     690    5     787    5     783    5     744    5     761    5     746    5     793    5     696    5     749    5     745    5     755    5     800    5     778    5     814    5     826    5     700    5     740    5     773    5     713    5     824    5     792    5     702    5     734    5     751    5     716    5     718    5     722    5     784    5     778    5     700    5     714    5     739    5     748    5     697    5     751    5     663    5     740    5
+      │    │    │    │    │                <--- 37 ----- 834 ----- 1623 ----- 2351 ----- 3101 ----- 3828 ----- 4598 ----- 5401 ----- 6176 ----- 6868 ----- 7613 ----- 8479 ----- 9230 ----- 9972 ----- 10613 ----- 11500 ----- 12282 ----- 13103 ----- 13624 ----- 14312 ----- 14962 ----- 15520 ----- 16319 ----- 17109 ----- 17780 ----- 18543 ----- 19408 ----- 20250 ----- 20984 ----- 21848 ----- 22551 ----- 23355 ----- 24174 ----- 24809 ----- 25567 ----- 26196 ----- 26868 ----- 27710 ----- 28412 ----- 29276 ----- 29889 ----- 30645 ----- 31264 ----- 32139 ----- 32736 ----- 33527 ----- 34418 ----- 35091 ----- 35709 ----- 36498 ----- 37150 ----- 37861 ----- 38677 ----- 39517 ----- 40316 ----- 40975 ----- 41782 ----- 42569 ----- 43565 ----- 44224 ----- 44867 ----- 45399 ----- 46231 ----- 46902 ----- 47630 ----- 48291 ----- 49087 ----- 49829 ----- 50580 ----- 51538 ----- 52375 ----- 53092 ----- 53932 ----- 54656 ----- 55442 ----- 56274 ----- 57121 ----- 57929 ----- 58888 ----- 59705 ----- 60460 ----- 61282 ----- 62031 ----- 62922 ----- 63785 ----- 64593 ----- 65311 ----- 66024 ----- 66875 ----- 67556 ----- 68331 ----- 68808 ----- 69564 ----- 70239 ----- 70983 ----- 71744 ----- 72478 ----- 73223 ----- 74088 ----- 74988 ----- 75868 ----- 76539 ----- 77203 ----- 78061 ----- 78901 ----- 79510 ----- 80417 ----- 81071 ----- 81826 ----- 82553 ----- 83191 ----- 83828 ----- 84485 ----- 85386 ----- 86159 ----- 86868 ----- 87628 ----- 88463 ----- 89240 ----- 90002 ----- 90822 ----- 91553 ----- 92367 ----- 93152 ----- 93929 ----- 94656 ----- 95470 ----- 96061 ----- 96720 ----- 97355 ----- 98200 ----- 98998 ----- 99573 ----- 100219 ----- 100975 ----- 101795 ----- 102620 ----- 103384 ----- 104044 ----- 104635 ----- 105426 ----- 106083 ----- 107049 ----- 107925 ----- 108715 ----- 109740 ----- 110496 ----- 111128 ----- 111807 ----- 112451 ----- 113184 ----- 113866 ----- 114619 ----- 115556 ----- 116344 ----- 117029 ----- 117859 ----- 118626 ----- 119515 ----- 120258 ----- 121021 ----- 121802 ----- 122505 ----- 123136 ----- 123953 ----- 124763 ----- 125501 ----- 126271 ----- 127012 ----- 127841 ----- 128483 ----- 129230 ----- 129970 ----- 130729 ----- 131569 ----- 132370 ----- 133235 ----- 134122 ----- 134773 ----- 135503 ----- 136294 ----- 136971 ----- 137854 ----- 138681 ----- 139336 ----- 140055 ----- 140806 ----- 141489 ----- 142177 ----- 142873 ----- 143685 ----- 144486 ----- 145138 ----- 145817 ----- 146545 ----- 147291 ----- 147936 ----- 148687 ----- 149260 ----- 149990
       │    │    │    │    │   histogram(5)=  0          1          1.5e+05          1
-      │    │    │    │    │                <--- '10-101-662-5213' --------- '34-993-352-7344'
-      │    │    │    │    │   histogram(6)=  0          15          1.4997e+05       15
-      │    │    │    │    │                <--- -998.5999755859375 ------------ 9999.5703125
+      │    │    │    │    │                <--- '10-104-665-3850' --------- '34-996-464-1615'
+      │    │    │    │    │   histogram(6)=  0    15    1.4997e+05        15
+      │    │    │    │    │                <--- -997.5 ------------ 9999.6904296875
       │    │    │    │    ├── key: (1)
       │    │    │    │    └── fd: (1)-->(5,6)
       │    │    │    └── filters
@@ -119,17 +119,17 @@ sort
       │    │    │                        │    ├── save-table-name: q22_select_8
       │    │    │                        │    ├── columns: c_phone:15(char!null) c_acctbal:16(float!null)
       │    │    │                        │    ├── immutable
-      │    │    │                        │    ├── stats: [rows=45456.062, distinct(15)=45456.062, null(15)=0, distinct(16)=45456.062, null(16)=0]
-      │    │    │                        │    │   histogram(16)=  0   0   45451       5
-      │    │    │                        │    │                 <--- 0.0 ------- 9999.5703125
+      │    │    │                        │    ├── stats: [rows=45460.6578, distinct(15)=45460.6578, null(15)=0, distinct(16)=45460.6578, null(16)=0]
+      │    │    │                        │    │   histogram(16)=  0   0   45456         5
+      │    │    │                        │    │                 <--- 0.0 ------- 9999.6904296875
       │    │    │                        │    ├── scan customer
       │    │    │                        │    │    ├── save-table-name: q22_scan_9
       │    │    │                        │    │    ├── columns: c_phone:15(char!null) c_acctbal:16(float!null)
       │    │    │                        │    │    └── stats: [rows=150000, distinct(15)=150000, null(15)=0, distinct(16)=140426, null(16)=0]
       │    │    │                        │    │        histogram(15)=  0          1          1.5e+05          1
-      │    │    │                        │    │                      <--- '10-101-662-5213' --------- '34-993-352-7344'
-      │    │    │                        │    │        histogram(16)=  0          15          1.4997e+05       15
-      │    │    │                        │    │                      <--- -998.5999755859375 ------------ 9999.5703125
+      │    │    │                        │    │                      <--- '10-104-665-3850' --------- '34-996-464-1615'
+      │    │    │                        │    │        histogram(16)=  0    15    1.4997e+05        15
+      │    │    │                        │    │                      <--- -997.5 ------------ 9999.6904296875
       │    │    │                        │    └── filters
       │    │    │                        │         ├── c_acctbal:16 > 0.0 [type=bool, outer=(16), constraints=(/16: [/5e-324 - ]; tight)]
       │    │    │                        │         └── substring(c_phone:15, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31') [type=bool, outer=(15), immutable]
@@ -221,8 +221,8 @@ column_names  row_count  distinct_count  null_count
 {c_phone}     38120      38046           0
 ~~~~
 column_names  row_count_est  row_count_err  distinct_count_est  distinct_count_err  null_count_est  null_count_err
-{c_acctbal}   45456.00       1.19           45456.00            1.22                0.00            1.00
-{c_phone}     45456.00       1.19           45456.00            1.19                0.00            1.00
+{c_acctbal}   45461.00       1.19           45461.00            1.22                0.00            1.00
+{c_phone}     45461.00       1.19           45461.00            1.19                0.00            1.00
 
 ----Stats for q22_scan_9----
 column_names  row_count  distinct_count  null_count

diff --git a/pkg/sql/opt/testutils/opttester/testfixtures/rewrite_stats.sh b/pkg/sql/opt/testutils/opttester/testfixtures/rewrite_stats.sh
@@ -20,7 +20,7 @@ FILENAME=$3
 
 tables=($($COCKROACH_BINARY sql --insecure --format=tsv \
  -e "USE $DATABASE; SELECT table_name FROM [SHOW TABLES] ORDER BY table_name;"))
-tables=("${tables[@]:3}")
+tables=("${tables[@]:1}")
 
 echo "Writing statistics to $FILENAME"
 echo "# Statistics for $DATABASE" > $FILENAME
@@ -32,14 +32,14 @@ do
     echo -n "ALTER TABLE \"$table\" INJECT STATISTICS '" >> $FILENAME
     # This command selects the statistics in JSON form for the current table. The output
     # is passed through the following sed commands before being written to the file:
-    # 1. Remove the first two lines to leave only the JSON value.
+    # 1. Remove the first line to leave only the JSON value.
     # 2. Replace duplicate double quotes with one double quote.
     # 3. Remove the double quote at the beginning of the JSON.
     # 4. Remove the double quote at the end of the JSON.
     # 5. Append '; to the last line.
     $COCKROACH_BINARY sql --insecure --format=tsv \
      -e "USE $DATABASE; SELECT jsonb_pretty(statistics) FROM [SHOW STATISTICS USING JSON FOR TABLE \"$table\"];" \
-     | sed '1,2d' | sed 's/""/"/g' | sed 's/^"\[/\[/g' | sed 's/\]"$/\]/g' | sed '$ s/$/'\'';/' >> $FILENAME
+     | sed '1d' | sed 's/""/"/g' | sed 's/^"\[/\[/g' | sed 's/\]"$/\]/g' | sed '$ s/$/'\'';/' >> $FILENAME
     echo "----" >> $FILENAME
     echo "" >> $FILENAME
 done