cleanup: use script to update external/googleapis (#7650)

Use a script to update the list of `.proto` files use by each library,
as well as the list of libraries each library depends on.

The more complicated case was the `google-cloud-cpp::logging_type_protos`
library, its name does not match the Bazel rules where it would generate
`*logging_type_type_protos`. I introduced an `INTERFACE` library for
backwards compatibility.

In the bigquery library we are exposing both the `v1beta1/` and `v1/`
versions of `connection.proto`. I will figure out how to clean that up
in a future PR.

ci/cloudbuild/builds/cmake-install.sh

@@ -53,11 +53,14 @@ expected_dirs+=(
   ./include/google/api
   ./include/google/bigtable/v2
   ./include/google/cloud/bigquery/connection
+  ./include/google/cloud/bigquery/connection/v1
   ./include/google/cloud/bigquery/connection/v1beta1
   ./include/google/cloud/bigquery/datatransfer
   ./include/google/cloud/bigquery/datatransfer/v1
   ./include/google/cloud/bigquery/logging
   ./include/google/cloud/bigquery/logging/v1
+  ./include/google/cloud/bigquery/reservation
+  ./include/google/cloud/bigquery/reservation/v1
   ./include/google/cloud/bigquery/v2
   ./include/google/cloud/bigtable/internal
   ./include/google/cloud/dialogflow

cmake/CompileProtos.cmake

@@ -330,7 +330,7 @@ macro (external_googleapis_install_pc_common target)
     # Examine the target LINK_LIBRARIES property, use that to pull the
     # dependencies between the google-cloud-cpp::* libraries.
     set(_target_pc_requires)
-    get_target_property(_target_deps ${target} LINK_LIBRARIES)
+    get_target_property(_target_deps ${target} INTERFACE_LINK_LIBRARIES)
     foreach (dep ${_target_deps})
         if ("${dep}" MATCHES "^google-cloud-cpp::")
             string(REPLACE "google-cloud-cpp::" "google_cloud_cpp_" dep

external/googleapis/protodeps/bigquery.deps

@@ -0,0 +1,10 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/iam/v1:iam_policy_proto
+@com_google_googleapis//google/iam/v1:options_proto
+@com_google_googleapis//google/iam/v1:policy_proto
+@com_google_googleapis//google/rpc:status_proto
+@com_google_googleapis//google/type:expr_proto

external/googleapis/protodeps/bigtable.deps

@@ -0,0 +1,11 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/iam/v1:iam_policy_proto
+@com_google_googleapis//google/iam/v1:options_proto
+@com_google_googleapis//google/iam/v1:policy_proto
+@com_google_googleapis//google/longrunning:operations_proto
+@com_google_googleapis//google/rpc:status_proto
+@com_google_googleapis//google/type:expr_proto

external/googleapis/protodeps/dialogflow.deps

@@ -0,0 +1,8 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/longrunning:operations_proto
+@com_google_googleapis//google/rpc:status_proto
+@com_google_googleapis//google/type:latlng_proto

external/googleapis/protodeps/iam.deps

@@ -0,0 +1,9 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/iam/v1:iam_policy_proto
+@com_google_googleapis//google/iam/v1:options_proto
+@com_google_googleapis//google/iam/v1:policy_proto
+@com_google_googleapis//google/type:expr_proto

external/googleapis/protodeps/logging.deps

@@ -0,0 +1,12 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:distribution_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:label_proto
+@com_google_googleapis//google/api:launch_stage_proto
+@com_google_googleapis//google/api:metric_proto
+@com_google_googleapis//google/api:monitored_resource_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/logging/type:type_proto
+@com_google_googleapis//google/rpc:status_proto

external/googleapis/protodeps/logging_type.deps

@@ -0,0 +1,2 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:http_proto

external/googleapis/protodeps/monitoring.deps

@@ -0,0 +1,13 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:distribution_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:label_proto
+@com_google_googleapis//google/api:launch_stage_proto
+@com_google_googleapis//google/api:metric_proto
+@com_google_googleapis//google/api:monitored_resource_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/longrunning:operations_proto
+@com_google_googleapis//google/rpc:status_proto
+@com_google_googleapis//google/type:calendar_period_proto

external/googleapis/protodeps/pubsub.deps

@@ -0,0 +1,5 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto

external/googleapis/protodeps/spanner.deps

@@ -0,0 +1,11 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/iam/v1:iam_policy_proto
+@com_google_googleapis//google/iam/v1:options_proto
+@com_google_googleapis//google/iam/v1:policy_proto
+@com_google_googleapis//google/longrunning:operations_proto
+@com_google_googleapis//google/rpc:status_proto
+@com_google_googleapis//google/type:expr_proto

external/googleapis/protodeps/speech.deps

@@ -0,0 +1,6 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/longrunning:operations_proto
+@com_google_googleapis//google/rpc:status_proto

external/googleapis/protodeps/storage.deps

@@ -0,0 +1,10 @@
+@com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
+@com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto
+@com_google_googleapis//google/iam/v1:iam_policy_proto
+@com_google_googleapis//google/iam/v1:options_proto
+@com_google_googleapis//google/iam/v1:policy_proto
+@com_google_googleapis//google/type:date_proto
+@com_google_googleapis//google/type:expr_proto

external/googleapis/protodeps/texttospeech.deps

@@ -1,5 +1,5 @@
-@com_google_googleapis//google/api:resource_proto
-@com_google_googleapis//google/api:field_behavior_proto
-@com_google_googleapis//google/api:client_proto
 @com_google_googleapis//google/api:annotations_proto
+@com_google_googleapis//google/api:client_proto
+@com_google_googleapis//google/api:field_behavior_proto
 @com_google_googleapis//google/api:http_proto
+@com_google_googleapis//google/api:resource_proto

external/googleapis/protolists/bigquery.list

@@ -0,0 +1,16 @@
+@com_google_googleapis//google/cloud/bigquery/connection/v1:connection.proto
+@com_google_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer.proto
+@com_google_googleapis//google/cloud/bigquery/datatransfer/v1:transfer.proto
+@com_google_googleapis//google/cloud/bigquery/logging/v1:audit_data.proto
+@com_google_googleapis//google/cloud/bigquery/reservation/v1:reservation.proto
+@com_google_googleapis//google/cloud/bigquery/storage/v1:arrow.proto
+@com_google_googleapis//google/cloud/bigquery/storage/v1:avro.proto
+@com_google_googleapis//google/cloud/bigquery/storage/v1:protobuf.proto
+@com_google_googleapis//google/cloud/bigquery/storage/v1:storage.proto
+@com_google_googleapis//google/cloud/bigquery/storage/v1:stream.proto
+@com_google_googleapis//google/cloud/bigquery/storage/v1:table.proto
+@com_google_googleapis//google/cloud/bigquery/v2:encryption_config.proto
+@com_google_googleapis//google/cloud/bigquery/v2:model.proto
+@com_google_googleapis//google/cloud/bigquery/v2:model_reference.proto
+@com_google_googleapis//google/cloud/bigquery/v2:standard_sql.proto
+@com_google_googleapis//google/cloud/bigquery/v2:table_reference.proto

external/googleapis/protolists/bigtable.list

@@ -0,0 +1,7 @@
+@com_google_googleapis//google/bigtable/admin/v2:bigtable_instance_admin.proto
+@com_google_googleapis//google/bigtable/admin/v2:bigtable_table_admin.proto
+@com_google_googleapis//google/bigtable/admin/v2:common.proto
+@com_google_googleapis//google/bigtable/admin/v2:instance.proto
+@com_google_googleapis//google/bigtable/admin/v2:table.proto
+@com_google_googleapis//google/bigtable/v2:bigtable.proto
+@com_google_googleapis//google/bigtable/v2:data.proto

external/googleapis/protolists/dialogflow.list

@@ -0,0 +1,21 @@
+@com_google_googleapis//google/cloud/dialogflow/v2:agent.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:answer_record.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:audio_config.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:context.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:conversation_event.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:conversation_profile.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:conversation.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:document.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:entity_type.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:environment.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:fulfillment.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:gcs.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:human_agent_assistant_event.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:intent.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:knowledge_base.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:participant.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:session_entity_type.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:session.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:validation_result.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:version.proto
+@com_google_googleapis//google/cloud/dialogflow/v2:webhook.proto

external/googleapis/protolists/iam.list

@@ -0,0 +1,4 @@
+@com_google_googleapis//google/iam/admin/v1:audit_data.proto
+@com_google_googleapis//google/iam/admin/v1:iam.proto
+@com_google_googleapis//google/iam/credentials/v1:common.proto
+@com_google_googleapis//google/iam/credentials/v1:iamcredentials.proto

external/googleapis/protolists/logging.list

@@ -0,0 +1,4 @@
+@com_google_googleapis//google/logging/v2:log_entry.proto
+@com_google_googleapis//google/logging/v2:logging_config.proto
+@com_google_googleapis//google/logging/v2:logging_metrics.proto
+@com_google_googleapis//google/logging/v2:logging.proto

external/googleapis/protolists/logging_type.list

@@ -0,0 +1,2 @@
+@com_google_googleapis//google/logging/type:http_request.proto
+@com_google_googleapis//google/logging/type:log_severity.proto

external/googleapis/protolists/monitoring.list

@@ -0,0 +1,17 @@
+@com_google_googleapis//google/monitoring/v3:alert.proto
+@com_google_googleapis//google/monitoring/v3:alert_service.proto
+@com_google_googleapis//google/monitoring/v3:common.proto
+@com_google_googleapis//google/monitoring/v3:dropped_labels.proto
+@com_google_googleapis//google/monitoring/v3:group.proto
+@com_google_googleapis//google/monitoring/v3:group_service.proto
+@com_google_googleapis//google/monitoring/v3:metric.proto
+@com_google_googleapis//google/monitoring/v3:metric_service.proto
+@com_google_googleapis//google/monitoring/v3:mutation_record.proto
+@com_google_googleapis//google/monitoring/v3:notification.proto
+@com_google_googleapis//google/monitoring/v3:notification_service.proto
+@com_google_googleapis//google/monitoring/v3:query_service.proto
+@com_google_googleapis//google/monitoring/v3:service.proto
+@com_google_googleapis//google/monitoring/v3:service_service.proto
+@com_google_googleapis//google/monitoring/v3:span_context.proto
+@com_google_googleapis//google/monitoring/v3:uptime.proto
+@com_google_googleapis//google/monitoring/v3:uptime_service.proto

external/googleapis/protolists/pubsub.list

@@ -0,0 +1,2 @@
+@com_google_googleapis//google/pubsub/v1:pubsub.proto
+@com_google_googleapis//google/pubsub/v1:schema.proto

external/googleapis/protolists/spanner.list

@@ -0,0 +1,12 @@
+@com_google_googleapis//google/spanner/admin/database/v1:backup.proto
+@com_google_googleapis//google/spanner/admin/database/v1:common.proto
+@com_google_googleapis//google/spanner/admin/database/v1:spanner_database_admin.proto
+@com_google_googleapis//google/spanner/admin/instance/v1:spanner_instance_admin.proto
+@com_google_googleapis//google/spanner/v1:commit_response.proto
+@com_google_googleapis//google/spanner/v1:keys.proto
+@com_google_googleapis//google/spanner/v1:mutation.proto
+@com_google_googleapis//google/spanner/v1:query_plan.proto
+@com_google_googleapis//google/spanner/v1:result_set.proto
+@com_google_googleapis//google/spanner/v1:spanner.proto
+@com_google_googleapis//google/spanner/v1:transaction.proto
+@com_google_googleapis//google/spanner/v1:type.proto

external/googleapis/protolists/speech.list

@@ -0,0 +1 @@
+@com_google_googleapis//google/cloud/speech/v1:cloud_speech.proto

external/googleapis/protolists/storage.list

@@ -0,0 +1 @@
+@com_google_googleapis//google/storage/v2:storage.proto

external/googleapis/update_libraries.sh

@@ -16,24 +16,67 @@
 set -euo pipefail
 
 declare -A -r LIBRARIES=(
+  ["bigquery"]="$(
+    printf ",%s" \
+      "@com_google_googleapis//google/cloud/bigquery/v2:bigquery_proto" \
+      "@com_google_googleapis//google/cloud/bigquery/storage/v1:storage_proto" \
+      "@com_google_googleapis//google/cloud/bigquery/reservation/v1:reservation_proto" \
+      "@com_google_googleapis//google/cloud/bigquery/logging/v1:logging_proto" \
+      "@com_google_googleapis//google/cloud/bigquery/datatransfer/v1:datatransfer_proto" \
+      "@com_google_googleapis//google/cloud/bigquery/connection/v1:connection_proto"
+  )"
+  ["bigtable"]="$(
+    printf ",%s" \
+      "@com_google_googleapis//google/bigtable/v2:bigtable_proto" \
+      "@com_google_googleapis//google/bigtable/admin/v2:admin_proto"
+  )"
+  ["dialogflow"]="@com_google_googleapis//google/cloud/dialogflow/v2:dialogflow_proto"
+  ["iam"]="$(
+    printf ",%s" \
+      "@com_google_googleapis//google/iam/credentials/v1:credentials_proto" \
+      "@com_google_googleapis//google/iam/admin/v1:admin_proto"
+  )"
+  ["logging_type"]="@com_google_googleapis//google/logging/type:type_proto"
+  ["logging"]="@com_google_googleapis//google/logging/v2:logging_proto"
+  ["monitoring"]="@com_google_googleapis//google/monitoring/v3:monitoring_proto"
+  ["pubsub"]="@com_google_googleapis//google/pubsub/v1:pubsub_proto"
+  ["spanner"]="$(
+    printf ",%s" \
+      "@com_google_googleapis//google/spanner/v1:spanner_proto" \
+      "@com_google_googleapis//google/spanner/admin/instance/v1:instance_proto" \
+      "@com_google_googleapis//google/spanner/admin/database/v1:database_proto"
+  )"
+  ["speech"]="@com_google_googleapis//google/cloud/speech/v1:speech_proto"
+  ["storage"]="@com_google_googleapis//google/storage/v2:storage_proto"
   ["texttospeech"]="@com_google_googleapis//google/cloud/texttospeech/v1:texttospeech_proto"
 )
 
 for library in "${!LIBRARIES[@]}"; do
   IFS=',' read -r -a rules <<<"${LIBRARIES[$library]}"
-  : >"external/googleapis/protolists/${library}.list"
-  : >"external/googleapis/protodeps/${library}.deps"
+  files=(
+    "external/googleapis/protolists/${library}.list"
+    "external/googleapis/protodeps/${library}.deps"
+  )
+  for file in "${files[@]}"; do
+    : >"${file}"
+  done
   for rule in "${rules[@]}"; do
+    if [[ -z "${rule}" ]]; then continue; fi
     path="${rule%:*}"
-    bazel --batch query --noshow_progress --noshow_loading_progress \
+    echo "=== $library $rule $path"
+    bazel query --noshow_progress --noshow_loading_progress \
       "deps(${rule})" |
       grep "${path}" |
       grep -E '\.proto$' \
         >>"external/googleapis/protolists/${library}.list"
-    bazel --batch query --noshow_progress --noshow_loading_progress \
+    bazel query --noshow_progress --noshow_loading_progress \
       "deps(${rule})" |
       grep "@com_google_googleapis//" | grep _proto |
       grep -v "${path}" \
         >>"external/googleapis/protodeps/${library}.deps"
   done
+  for file in "${files[@]}"; do
+    sort -u "${file}" >"${file}.sorted"
+    mv "${file}.sorted" "${file}"
+  done
 done