diff --git a/scripts/pantheon_cloud_footprint_script.py b/scripts/pantheon_cloud_footprint_script.py new file mode 100644 index 0000000..a9ec3d1 --- /dev/null +++ b/scripts/pantheon_cloud_footprint_script.py @@ -0,0 +1,215 @@ +import subprocess +import argparse + +parser = argparse.ArgumentParser(description='GCP Footprint Scanner') +parser.add_argument('-c', '--cloud', action='append', dest='clouds', + help='Specify which clouds to scan. Can be GCP, Github or AWS', required=False, default=[]) + +parser.add_argument('-a', '--aws-account', action='append', dest='aws_accounts', help='Specify AWS account', required=False) +parser.add_argument('-p', '--gcp-project', action='append', dest='gcp_projects', help='Specify GCP project', required=False) +parser.add_argument('-o', '--github-org', action='append', dest='github_orgs', help='Specify GitHub organizations', required=False) + +parser.add_argument('-d', '--debug', action="store_true", dest='debug_logging', help='Enable debug logging', required=False) +parser.add_argument('-v', '--verbose', action="store_true", dest='verbose_logging', help='Enable verbose logging', required=False) +args = parser.parse_args() + +gcp_totals = { + "Cloud Functions": 0, + "Cloud Run Services": 0, + "App Engine Services": 0, + "Compute Instances": 0, + "GKE Clusters": 0, + "Notebook Instances": 0, + "Redis Instances": 0, + "AlloyDB Instances": 0, + "CloudSQL Instances": 0, + "BigTable Instances": 0, + "Dataflow Jobs": 0, + "BigQuery Datasets": 0, + "BigTable Tables": 0, + "GCS Buckets": 0 +} + +github_totals = { + "Repositories": 0, +} + +def filter_line(line): + if "DeprecationWarning:" in line or "import pipes" in line: + return True + return False + + +def count_lines(output): + splitlines = output.splitlines() + return len([l for l in splitlines if not filter_line(l)]) + + +# Function to safely execute gcloud commands and ignore some errors +def safe_call(command): + try: + output = subprocess.check_output(command, shell=True, stderr=subprocess.STDOUT, text=True) + return output.strip() + except subprocess.CalledProcessError as e: + if "SERVICE_DISABLED" in e.output or "PERMISSION_DENIED" in e.output: + return '' + else: + raise + + +# Function to safely execute gcloud commands and ignore SERVICE_DISABLED errors +def count_call(command): + if args.debug_logging: + print(f"Executing command: {command}") + result = safe_call(command) + lines = count_lines(result) + if args.verbose_logging: + print(f"Got result: {result} with count {lines}") + return lines + + +def count_gcp_regional_resources(regions, project): + notebook_instances = 0 + redis_instances = 0 + dataflow_jobs = 0 + + for region in regions: + notebook_instances += count_call( + f"gcloud --quiet notebooks instances list --project={project} --location={region} --format=\"value(name)\"") + redis_instances += count_call( + f"gcloud --quiet redis instances list --project={project} --region={region} --format=\"value(name)\"") + dataflow_jobs += count_call( + f"gcloud --quiet dataflow jobs list --project={project} --region={region} --format=\"value(id)\"") + + return { + "Notebook Instances": notebook_instances, + "Redis Instances": redis_instances, + "Dataflow Jobs": dataflow_jobs, + } + +def report_counts(aca, counts): + print(f"Scanned resources for: {aca}") + for resource, count in counts.items(): + print(f"{resource}: {count}") + print("-----------------------------------") + +def count_gcp_resources(regions, project): + # Count resources + cloud_functions = count_call(f"gcloud --quiet functions list --project={project} --format=\"value(name)\"") + cloud_run_services = count_call( + f"gcloud --quiet run services list --project={project} --platform managed --format=\"value(metadata.name)\"") + app_engine_services = count_call(f"gcloud --quiet app services list --project={project} --format=\"value(id)\"") + compute_instances = count_call( + f"gcloud --quiet compute instances list --project={project} --format=\"value(name)\"") + gke_clusters = count_call(f"gcloud --quiet container clusters list --project={project} --format=\"value(name)\"") + + # regional resources + regional_counts = count_gcp_regional_resources(regions, project) + notebook_instances = regional_counts["Notebook Instances"] + redis_instances = regional_counts["Redis Instances"] + dataflow_jobs = regional_counts["Dataflow Jobs"] + + alloydb_instances = count_call( + f"gcloud --quiet alloydb instances list --project={project} --format=\"value(name)\"") + cloudsql_instances = count_call(f"gcloud --quiet sql instances list --project={project} --format=\"value(name)\"") + bigquery_datasets = count_call(f"bq ls -q --project_id={project} --format=csv") + gcs_buckets = count_call(f"gsutil -q ls -p {project}") + + bigtable_tables_output = safe_call(f"gcloud --quiet bigtable instances list --project={project} --format=\"value(name)\"") + bigtable_instances = count_lines(bigtable_tables_output) + bigtable_tables = 0 + for instance in bigtable_tables_output.splitlines(): + bigtable_tables += count_call(f"gcloud --quiet bigtable tables list --project={project} --instance={instance} --format=\"value(name)\"") + + counts = { + "Cloud Functions": cloud_functions, + "Cloud Run Services": cloud_run_services, + "App Engine Services": app_engine_services, + "Compute Instances": compute_instances, + "GKE Clusters": gke_clusters, + "Notebook Instances": notebook_instances, + "Redis Instances": redis_instances, + "Dataflow Jobs": dataflow_jobs, + "AlloyDB Instances": alloydb_instances, + "CloudSQL Instances": cloudsql_instances, + "BigTable Instances": bigtable_instances, + "BigTable Tables": bigtable_tables, + "BigQuery Datasets": bigquery_datasets, + "GCS Buckets": gcs_buckets + } + + report_counts(project, counts) + return counts + +def count_github_resources(org): + repos = count_call(f"gh repo list {org} -L 10000 --no-archived") + + counts = { + "Repositories": repos, + } + report_counts(org, counts) + return counts + + +def scan_gcp(): + projects = args.gcp_projects + if not args.gcp_projects or len(projects) == 0: + projects_output = safe_call("gcloud --quiet projects list --format=\"value(projectId)\"") + projects = projects_output.splitlines() + + if len(projects) == 0: + print("No projects found.") + exit() + + regions_output = safe_call( + f"gcloud --quiet compute regions list --format=\"value(name)\" --project={projects[0]}") + regions = regions_output.splitlines() + + print(f"Starting GCP resource scanning for {len(projects)} projects") + print("-----------------------------------") + # Aggregate resources for each project + for project in projects: + counts = count_gcp_resources(regions, project) + for resource, count in counts.items(): + gcp_totals[resource] += count + + # Print totals + print(f"Total GCP resource count in {len(projects)} projects") + print("============") + for resource, total in gcp_totals.items(): + print(f"Total {resource}: {total}") + + +def scan_github(): + orgs = args.github_orgs + if not args.github_orgs or len(orgs) == 0: + orgs_output = safe_call("gh org list") + orgs = orgs_output.splitlines() + + if len(orgs) == 0: + print("No organizations found.") + exit() + + print(f"Starting GitHub resource scanning for {len(orgs)} organizations") + print("-----------------------------------") + # Aggregate resources for each project + for org in orgs: + counts = count_github_resources(org) + for resource, count in counts.items(): + github_totals[resource] += count + + # Print totals + print(f"Total Github resource count in {len(orgs)} organizations") + print("============") + for resource, total in github_totals.items(): + print(f"Total {resource}: {total}") + + +def run_cloud_module(cloud): + return any(cloud.lower() == cm.lower() for cm in args.clouds) + + +if run_cloud_module("GCP"): + scan_gcp() +if run_cloud_module("GitHub"): + scan_github()