Add ray-logs-sidecar

.github/scripts/version

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+# Verify clean tree
+[ -z "$(git status --porcelain)" ] || ( echo >&2 "Git tree not clean!" && exit 1 )
+
+# Use gitcalver if we are on main
+if [ "$(git branch --show-current)" = main ]; then
+    TZ=UTC
+    # Prints version in the following format: <DAY>.<COMMIT#>_<SHA>
+    # From docker docs: A tag name may contain lowercase and uppercase characters,
+    # digits, underscores, periods and dashes. A tag name may not start with a period
+    # or a dash and may contain a maximum of 128 characters.
+    git log --pretty="format:%cd %h" --date="format-local:%Y%m%d" | \
+        awk 'NR==1{d=$1;h=$2}{if(d==$1)c++;else exit}END{print d"."c"_"h}'
+else
+    printf $(git branch --show-current | tr / -)_$(git rev-parse --short HEAD)
+fi

.github/workflows/ray-logs-sidecar.yml

@@ -0,0 +1,43 @@
+name: Build & Push Ray Logs Sidecar Image
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/ray-logs-sidecar.yml
+      - ray-logs-sidecar/**
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v2
+        if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+        with:
+          registry: https://ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - id: image-names
+        uses: docker/metadata-action@v3
+        with:
+          images: ghcr.io/${{ github.repository_owner }}/ray-logs-sidecar
+          tags: |
+            type=raw,value=latest,enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+            type=sha,format=long
+      - uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' }}
+          tags: ${{ steps.image-names.outputs.tags }}

README.md

@@ -1 +1,10 @@
-# containers
+# Containers
+
+This repository contains Dockerfiles for useful containers that can be used with [Flyte](https://github.com/flyteorg/flyte).
+
+The following containers are currently available:
+
+| Container          | Description                                             |
+| ------------------ | ------------------------------------------------------- |
+| `ray-logs-sidecar` | Captures and exposes Ray job logs to container's stdout |
+

ray-logs-sidecar/Dockerfile

@@ -0,0 +1,3 @@
+FROM fluent/fluent-bit:2.1.10
+LABEL org.opencontainers.image.source="https://github.com/unionai-oss/containers"
+COPY fluent-bit.conf strip-ansi-colors.lua /fluent-bit/etc/

ray-logs-sidecar/fluent-bit.conf

@@ -0,0 +1,17 @@
+[SERVICE]
+    Log_Level error
+[INPUT]
+    Name tail
+    Path /tmp/ray/session_latest/logs/job-driver-*
+    Refresh_Interval 1
+[FILTER]
+    Name    lua
+    Match   *
+    script  strip-ansi-colors.lua
+    call    strip_ansi_colors
+[OUTPUT]
+    Name file
+    Match *
+    File /dev/stdout
+    Format template
+    Template {log}

ray-logs-sidecar/strip-ansi-colors.lua

@@ -0,0 +1,11 @@
+-- From: https://github.com/fluent/fluent-bit/discussions/6151#discussioncomment-4392962
+function strip_ansi_colors(tag, timestamp, record)
+  if type(record["log"]) == "string" then
+    -- strip ansi https://stackoverflow.com/a/49209650/368691
+    record["log"] = record["log"]:gsub('[\27\155][][()#;?%d]*[A-PRZcf-ntqry=><~]', '')
+
+    return 2, 0, record
+  end
+
+  return 0, 0, 0
+end