generate

#!/bin/bash

# A script for generating `namespaces.yaml` and the associated LookML.
# This repository builds namespaces.yaml from Glean applications and
# `custom-namespaces.yaml`, and then generates files and LookML that
# match the specification in `namespaces.yaml` and table definitions
# in BigQuery. The resulting LookML is pushed to our `looker-hub`
# repository.
#
# Environment variables:
#   GIT_SSH_KEY_BASE64:       A base64-encoded ssh secret key with permissions to push
#                             to looker-hub and looker-spoke-default.
#   HUB_REPO_URL:             The URL to the looker-hub repository.
#                             Requires the SSH format, e.g. git@github.com:user/repo.git.
#   HUB_BRANCH_SOURCE:        The source branch for generating LookML.
#                             Defaults to 'base'. Files present the source
#                             branch will remain unchanged by generation.
#   HUB_BRANCH_PUBLISH:       The destination branch for publishing LookML.
#                             Defaults to 'test-lookml-generation'. If the
#                             branch doesn't exist, creates it from source.
#   SPOKE_REPO_URL:           The URL to the looker-spoke-default repository.
#                             Requires the SSH format, e.g. git@github.com:user/repo.git.
#   PRIVATE_SPOKE_REPO_URL:   The URL to the looker-spoke-private repository.
#                             Requires the SSH format, e.g. git@github.com:user/repo.git.
#   SPOKE_BRANCH_WORKING:     The working branch for the spoke project.
#                             This is the branch that changes will be
#                             published to, and a PR opened to merge
#                             them in to SPOKE_BRANCH_PUBLISH.
#                             Defaults to SPOKE_BRANCH_PUBLISH-working.
#   SPOKE_BRANCH_PUBLISH:     The publish branch for the spoke project.
#                             A PR to merge the changes from SPOKE_BRANCH_WORKING
#                             to here will be opened. This branch also serves as
#                             the source branch.
#   LOOKER_INSTANCE_URI:      The URI of our looker instance. Defaults to dev.
#   LOOKER_API_CLIENT_ID:     Client ID for Looker access. If unset, does
#                             not run `generator content`.
#   LOOKER_API_CLIENT_SECRET: Client Secret for Looker access. If unset, does
#                             not run `generator content`.
#   GITHUB_ACCESS_TOKEN:      Access token for Github. Needs read and write
#                             access to repos. Not required in dev.
#   UPDATE_DEV_BRANCHES:      Whether or not the dev branches should be updated.
#                             This should only happen in production.
#   UPDATE_SPOKE_BRANCHES:    Whether or not pull-requests should be opened against
#                             branches in spoke projects.
#                             This should only happen in production.
#
# Example usage:
#   export GIT_SSH_KEY_BASE64=$(cat ~/.ssh/id_rsa | base64)
#   make build && make run

HUB_REPO_URL=${HUB_REPO_URL:-"git@github.com:mozilla/looker-hub.git"}
HUB_BRANCH_SOURCE=${HUB_BRANCH_SOURCE:-"base"}
HUB_BRANCH_PUBLISH=${HUB_BRANCH_PUBLISH:-"main-nonprod"}

SPOKE_REPO_URL=${SPOKE_REPO_URL:-"git@github.com:mozilla/looker-spoke-default.git"}
PRIVATE_SPOKE_REPO_URL=${PRIVATE_SPOKE_REPO_URL:-"git@github.com:mozilla/looker-spoke-private.git"}
SPOKE_BRANCH_PUBLISH=${SPOKE_BRANCH_PUBLISH:-"main-nonprod"}
SPOKE_BRANCH_WORKING=${SPOKE_BRANCH_WORKING:-"${SPOKE_BRANCH_PUBLISH}-working"}

LOOKER_INSTANCE_URI=${LOOKER_INSTANCE_URI:-"https://mozilladev.cloud.looker.com"}
UPDATE_DEV_BRANCHES=${UPDATE_DEV_BRANCHES:-"false"}
UPDATE_SPOKE_BRANCHES=${UPDATE_SPOKE_BRANCHES:-"false"}

function setup_git_auth() {
    # Configure the container for pushing to github.

    if [[ -z "$GIT_SSH_KEY_BASE64" ]]; then
        echo "Missing secret key" 1>&2
        exit 1
    fi

    git config --global user.name "Generated LookML Creator"
    git config --global user.email "dataops+looker@mozilla.com"

    mkdir -p "$HOME/.ssh"

    echo "$GIT_SSH_KEY_BASE64" | base64 --decode > "$HOME"/.ssh/id_ed25519
    # Makes the future git-push non-interactive
    ssh-keyscan github.com > "$HOME"/.ssh/known_hosts

    chown -R "$(id -u):$(id -g)" "$HOME/.ssh"
    chmod 700 "$HOME/.ssh"
    chmod 700 "$HOME/.ssh/id_ed25519"

    # add private key to the ssh agent to prompt for password once
    eval "$(ssh-agent)"
    ssh-add
}

function setup_github_auth() {
  # Configure Github CLI auth.

  if [[ -z "$GITHUB_ACCESS_TOKEN" ]]; then
    gh auth login
  else
    gh auth login --with-token <<< "$GITHUB_ACCESS_TOKEN"
  fi
}

function setup_hub() {
    # Checkout looker-hub and changes directory to prepare for
    # LookML generation.
    pushd .

    cd /app
    [[ -d looker-hub ]] && rm -rf looker-hub
    git clone "$HUB_REPO_URL"
    cd looker-hub
    git fetch --all
    # If publish branch doesn't exist, create it from main
    git checkout "$HUB_BRANCH_PUBLISH" || (git checkout main && git checkout -b "$HUB_BRANCH_PUBLISH")
    git checkout "$HUB_BRANCH_SOURCE"

    popd
}

function setup_spoke() {
    # Checkout looker-spoke-default and changes directory to prepare for
    # LookML generation. Create publish branch if non-existent.
    pushd .
    spoke="$1"
    spoke_url="$2"

    cd /app
    [[ -d $spoke ]] && rm -rf $spoke
    git clone "$spoke_url"
    cd $spoke
    git fetch --all
    git checkout $SPOKE_BRANCH_PUBLISH || (git checkout main && git checkout -b $SPOKE_BRANCH_PUBLISH)
    git branch -D $SPOKE_BRANCH_WORKING || true # delete working branch if it exists
    git checkout -b $SPOKE_BRANCH_WORKING

    popd
}

function setup_spokes() {
  setup_spoke "looker-spoke-default" $SPOKE_REPO_URL
  setup_spoke "looker-spoke-private" $PRIVATE_SPOKE_REPO_URL
}

function check_files_and_commit() {
  # Add the new files and commit.
  # Use interactive mode to add untracked files
  # This also works when it's untracked directories
  echo -e "a\n*\nu\n*\nq\n"|git add -i
  git commit -m "Auto-push from LookML generation" \
    || echo "Nothing to commit"
}

function generate_hub_commit() {
  # Generate commit on publish branch with
  # generated LookML.

  pushd .
  cd /app

  HUB_DIR="looker-hub"
  NAMESPACE_DISALLOWLIST="/app/lookml-generator/namespaces-disallowlist.yaml"
  CUSTOM_NAMESPACES_FILENAME="/app/lookml-generator/custom-namespaces.yaml"
  GENERATED_SQL_URI="https://github.com/mozilla/bigquery-etl/archive/generated-sql.tar.gz"
  APP_LISTINGS_URI="https://probeinfo.telemetry.mozilla.org/v2/glean/app-listings"

  # Generate namespaces.yaml and LookML
  lookml-generator namespaces \
    --custom-namespaces $CUSTOM_NAMESPACES_FILENAME \
    --generated-sql-uri $GENERATED_SQL_URI \
    --app-listings-uri $APP_LISTINGS_URI \
    --disallowlist $NAMESPACE_DISALLOWLIST
  lookml-generator lookml \
    --namespaces "namespaces.yaml" \
    --target-dir $HUB_DIR

  cd $HUB_DIR

  check_files_and_commit

  # Checkout main. Match it with source branch.
  git checkout "$HUB_BRANCH_PUBLISH"
  find . -mindepth 1 -maxdepth 1 -not -name .git -exec rm -rf {} +
  git checkout "$HUB_BRANCH_SOURCE" -- *
  git commit --all \
    --message "Auto-push from LookML generation" \
    || echo "Nothing to commit"

  popd
}

function update_dev_branches() {
  # Reset all dev branches to main

  pushd .
  cd /app/looker-hub

  dev_branches_file="/app/lookml-generator/bin/dev_branches"
  while read branch; do
    git checkout $branch
    git reset --hard main
    git push -f origin $branch
  done < $dev_branches_file

  popd
}

function generate_spoke_commits() {
  # Generate commit on spoke publish branch
  # with generated LookML.

  pushd .
  cd /app

  lookml-generator update-spoke \
    --namespaces "namespaces.yaml" \
    --spoke-dir "/app"

  cd "/app/looker-spoke-default"
  check_files_and_commit

  cd "/app/looker-spoke-private"
  check_files_and_commit

  popd
}

function hit_looker_webhooks() {
  # These webhooks ensure production is up-to-date.
  # See https://help.looker.com/hc/en-us/articles/360001288268-Deploy-Webhook-Pulling-From-Remote-Git-Repository
  curl "$LOOKER_INSTANCE_URI/webhooks/projects/looker-hub/deploy"
  curl "$LOOKER_INSTANCE_URI/webhooks/projects/spoke-default/deploy"
  curl "$LOOKER_INSTANCE_URI/webhooks/projects/spoke-private/deploy"
}

function push_and_open_spoke_pull_request() {
  # Open a PR to merge the changes from working branch
  # in to the publish branch.

  git push -f || git push -f --set-upstream origin "$SPOKE_BRANCH_WORKING"

  # https://bugzilla.mozilla.org/show_bug.cgi?id=1774030
  # could not request reviewer: 'mozilla/data-looker' not found
  # git diff --quiet $SPOKE_BRANCH_PUBLISH...$SPOKE_BRANCH_WORKING ||
  #   gh pr create \
  #   --title "Auto-push from LookML Generator" \
  #   --reviewer mozilla/data-looker \
  #   --base $SPOKE_BRANCH_PUBLISH \
  #   --head $SPOKE_BRANCH_WORKING \
  #   --body ""

  git diff --quiet $SPOKE_BRANCH_PUBLISH...$SPOKE_BRANCH_WORKING ||
    gh pr create \
    --title "Auto-push from LookML Generator" \
    --base $SPOKE_BRANCH_PUBLISH \
    --head $SPOKE_BRANCH_WORKING \
    --body ""
}

function main() {
  set -e  # stop if any statement returns a non-zero exit code
  pushd .
  cd /app

  set +x  # don't print these commands
  setup_git_auth
  setup_github_auth

  # Set up hub and commit
  set -x  # print these commands
  setup_hub
  generate_hub_commit

  # Publish hub
  cd /app/looker-hub
  git push || git push --set-upstream origin "$HUB_BRANCH_PUBLISH"

  # Update dev branches
  if [ "$UPDATE_DEV_BRANCHES" = "true" ] ; then
    update_dev_branches
  fi

  # Update branches on spoke-default and spoke-private
  if [ "$UPDATE_SPOKE_BRANCHES" = "true" ] ; then
    # Set up spokes and commit
    setup_spokes
    generate_spoke_commits

    # Publish spoke - force push to working branch
    cd /app/looker-spoke-default
    push_and_open_spoke_pull_request

    cd /app/looker-spoke-private
    push_and_open_spoke_pull_request

    # Update Looker content
    hit_looker_webhooks
  fi

  popd
}

main "$@"