From 30f3356fc2b1be8ba4036d9223cc7a713366d868 Mon Sep 17 00:00:00 2001 From: igolaizola <11333576+igolaizola@users.noreply.github.com> Date: Wed, 26 Jul 2023 00:20:28 +0200 Subject: [PATCH] First implementation --- .github/FUNDING.yml | 2 + .github/workflows/ci.yml | 26 ++ .github/workflows/release.yml | 29 ++ .gitignore | 11 +- .goreleaser.yml | 14 + Dockerfile | 15 + Makefile | 74 +++++ README.md | 143 ++++++++- cmd/vidai/main.go | 213 +++++++++++++ go.mod | 5 + go.sum | 2 + internal/ratelimit/ratelimit.go | 47 +++ pkg/runway/runway.go | 514 ++++++++++++++++++++++++++++++++ vidai.go | 263 ++++++++++++++++ 14 files changed, 1351 insertions(+), 7 deletions(-) create mode 100644 .github/FUNDING.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 .goreleaser.yml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 cmd/vidai/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/ratelimit/ratelimit.go create mode 100644 pkg/runway/runway.go create mode 100644 vidai.go diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..3b69287 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,2 @@ +github: igolaizola +custom: ["https://ko-fi.com/igolaizola", "https://buymeacoffee.com/igolaizola", "https://paypal.me/igolaizola"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cf01225 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +name: ci + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup go + uses: actions/setup-go@v4 + with: + go-version-file: 'go.mod' + - name: Build + run: go build -v ./... + - name: Lint + uses: golangci/golangci-lint-action@v3 + - name: Test + run: go test -v ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c2af095 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: goreleaser + +on: + push: + tags: + - '*' + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + cache: true + - uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 3b735ec..b443ae4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,10 @@ -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib +bin # Test binary, built with `go test -c` *.test @@ -17,5 +15,8 @@ # Dependency directories (remove the comment below to include it) # vendor/ -# Go workspace file -go.work +# Other +.history +.vscode +*.conf +logs/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..4fb524d --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,14 @@ +builds: + - id: vidai + binary: vidai + main: ./cmd/vidai + goarch: + - amd64 + - arm64 + - arm +archives: + - id: vidai + builds: + - vidai + format: zip + name_template: 'vidai_{{ .Version }}_{{- if eq .Os "darwin" }}macos{{- else }}{{ .Os }}{{ end }}_{{ .Arch }}' diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f59938b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +# builder image +FROM golang:alpine as builder +ARG TARGETPLATFORM +COPY . /src +WORKDIR /src +RUN apk add --no-cache make bash git +RUN make app-build PLATFORMS=$TARGETPLATFORM + +# running image +FROM alpine +WORKDIR /home +COPY --from=builder /src/bin/vidai-* /bin/vidai + +# executable +ENTRYPOINT [ "/bin/vidai" ] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3e7190d --- /dev/null +++ b/Makefile @@ -0,0 +1,74 @@ +#!/bin/bash + +SHELL = /bin/bash +PLATFORMS ?= linux/amd64 darwin/amd64 windows/amd64 +IMAGE_PREFIX ?= igolaizola +REPO_NAME ?= vidai +COMMIT_SHORT ?= $(shell git rev-parse --verify --short HEAD) +VERSION ?= $(COMMIT_SHORT) +VERSION_NOPREFIX ?= $(shell echo $(VERSION) | sed -e 's/^[[v]]*//') + +# Build the binaries for the current platform +.PHONY: build +build: + os=$$(go env GOOS); \ + arch=$$(go env GOARCH); \ + PLATFORMS="$$os/$$arch" make app-build + +# Build the binaries +# Example: PLATFORMS=linux/amd64 make app-build +.PHONY: app-build +app-build: + @for platform in $(PLATFORMS) ; do \ + os=$$(echo $$platform | cut -f1 -d/); \ + arch=$$(echo $$platform | cut -f2 -d/); \ + arm=$$(echo $$platform | cut -f3 -d/); \ + arm=$${arm#v}; \ + ext=""; \ + if [ "$$os" == "windows" ]; then \ + ext=".exe"; \ + fi; \ + file=./bin/$(REPO_NAME)-$(VERSION_NOPREFIX)-$$(echo $$platform | tr / -)$$ext; \ + GOOS=$$os GOARCH=$$arch GOARM=$$arm CGO_ENABLED=0 \ + go build \ + -a -x -tags netgo,timetzdata -installsuffix cgo -installsuffix netgo \ + -ldflags " \ + -X main.Version=$(VERSION_NOPREFIX) \ + -X main.GitRev=$(COMMIT_SHORT) \ + " \ + -o $$file \ + ./cmd/$(REPO_NAME); \ + if [ $$? -ne 0 ]; then \ + exit 1; \ + fi; \ + chmod +x $$file; \ + done + +# Build the docker image +# Example: PLATFORMS=linux/amd64 make docker-build +.PHONY: docker-build +docker-build: + rm -rf bin; \ + @platforms=($(PLATFORMS)); \ + platform=$${platforms[0]}; \ + if [[ $${#platforms[@]} -ne 1 ]]; then \ + echo "Multi-arch build not supported"; \ + exit 1; \ + fi; \ + docker build --platform $$platform -t $(IMAGE_PREFIX)/$(REPO_NAME):$(VERSION) .; \ + if [ $$? -ne 0 ]; then \ + exit 1; \ + fi + +# Build the docker images using buildx +# Example: PLATFORMS="linux/amd64 darwin/amd64 windows/amd64" make docker-buildx +.PHONY: docker-buildx +docker-buildx: + @platforms=($(PLATFORMS)); \ + platform=$$(IFS=, ; echo "$${platforms[*]}"); \ + docker buildx build --platform $$platform -t $(IMAGE_PREFIX)/$(REPO_NAME):$(VERSION) . + +# Clean binaries +.PHONY: clean +clean: + rm -rf bin diff --git a/README.md b/README.md index 672473b..c95717f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,141 @@ -# vidai -Video generation using AI +# vidai 📹🤖 + +**vidai** generates videos using AI. + +This is a CLI tool for [RunwayML Gen-2](https://runwayml.com/) that adds some extra features on top of it. + +## 🚀 Features + + - Generate videos directly from the command line using a text or image prompt. + - Create or extend videos longer than 4 seconds by reusing the last frame of the video as the input for the next generation. + - Other handy tools to edit videos, like generating loops or resizing videos. + +## đŸ“Ļ Installation + +You can use the Golang binary to install **vidai**: + +```bash +go install github.com/igolaizola/vidai/cmd/vidai@latest +``` + +Or you can download the binary from the [releases](https://github.com/igolaizola/vidai/releases) + +## 📋 Requirements + +You need to have a [RunwayML](https://runwayml.com/) account and extract the token from the request authorization header using your browser's developer tools. + +To create extended videos, you need to have [ffmpeg](https://ffmpeg.org/) installed. + +## 🕹ī¸ Usage + +### Some examples + +Generate a video from an image prompt: + +```bash +vidai generate --token RUNWAYML_TOKEN --image car.jpg --output car.mp4 +``` + +Generate a video from a text prompt: + +```bash +vidai generate --token RUNWAYML_TOKEN --text "a car in the middle of the road" --output car.mp4 +``` + +Extend a video by reusing the last frame twice: + +```bash +vidai extend --input car.mp4 --output car-extended.mp4 --n 2 +``` + +Convert a video to a loop: + +```bash +vidai loop --input car.mp4 --output car-loop.mp4 +``` + +### Help + +Launch `vidai` with the `--help` flag to see all available commands and options: + +```bash +vidai --help +``` + +You can use the `--help` flag with any command to view available options: + +```bash +vidai generate --help +``` + +### How to launch commands + +Launch commands using a configuration file: + +```bash +vidai generate --config vidai.conf +``` + +```bash +# vidai.conf +token RUNWAYML_TOKEN +image car.jpg +output car.mp4 +extend 2 +``` + +Using environment variables (`VIDAI` prefix, uppercase and underscores): + +```bash +export VIDAI_TOKEN=RUNWAYML_TOKEN +export VIDAI_IMAGE="car.jpg" +export VIDAI_OUTPUT="car.mp4" +export VIDAI_EXTEND=2 +vidai generate +``` + +Using command line arguments: + +```bash +vidai generate --token RUNWAYML_TOKEN --image car.jpg --video car.mp4 --extend 2 +``` + +## ⚠ī¸ Disclaimer + +The automation of RunwayML accounts is a violation of their Terms of Service and will result in your account(s) being terminated. + +Read about RunwayML Terms of Service and Community Guidelines. + +vidai was written as a proof of concept and the code has been released for educational purposes only. The authors are released of any liabilities which your usage may entail. + +## 💖 Support + +If you have found my code helpful, please give the repository a star ⭐ + +Additionally, if you would like to support my late-night coding efforts and the coffee that keeps me going, I would greatly appreciate a donation. + +You can invite me for a coffee at ko-fi (0% fees): + +[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/igolaizola) + +Or at buymeacoffee: + +[![buymeacoffee](https://user-images.githubusercontent.com/11333576/223217083-123c2c53-6ab8-4ea8-a2c8-c6cb5d08e8d2.png)](https://buymeacoffee.com/igolaizola) + +Donate to my PayPal: + +[paypal.me/igolaizola](https://www.paypal.me/igolaizola) + +Sponsor me on GitHub: + +[github.com/sponsors/igolaizola](https://github.com/sponsors/igolaizola) + +Or donate to any of my crypto addresses: + + - BTC `bc1qvuyrqwhml65adlu0j6l59mpfeez8ahdmm6t3ge` + - ETH `0x960a7a9cdba245c106F729170693C0BaE8b2fdeD` + - USDT (TRC20) `TD35PTZhsvWmR5gB12cVLtJwZtTv1nroDU` + - USDC (BEP20) / BUSD (BEP20) `0x960a7a9cdba245c106F729170693C0BaE8b2fdeD` + - Monero `41yc4R9d9iZMePe47VbfameDWASYrVcjoZJhJHFaK7DM3F2F41HmcygCrnLptS4hkiJARCwQcWbkW9k1z1xQtGSCAu3A7V4` + +Thanks for your support! diff --git a/cmd/vidai/main.go b/cmd/vidai/main.go new file mode 100644 index 0000000..92afad9 --- /dev/null +++ b/cmd/vidai/main.go @@ -0,0 +1,213 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "os/signal" + "runtime/debug" + "strings" + "time" + + "github.com/igolaizola/vidai" + "github.com/peterbourgon/ff/v3" + "github.com/peterbourgon/ff/v3/ffcli" +) + +// Build flags +var version = "" +var commit = "" +var date = "" + +func main() { + // Create signal based context + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + // Launch command + cmd := newCommand() + if err := cmd.ParseAndRun(ctx, os.Args[1:]); err != nil { + log.Fatal(err) + } +} + +func newCommand() *ffcli.Command { + fs := flag.NewFlagSet("vidai", flag.ExitOnError) + + return &ffcli.Command{ + ShortUsage: "vidai [flags] ", + FlagSet: fs, + Exec: func(context.Context, []string) error { + return flag.ErrHelp + }, + Subcommands: []*ffcli.Command{ + newVersionCommand(), + newGenerateCommand(), + newExtendCommand(), + newLoopCommand(), + }, + } +} + +func newVersionCommand() *ffcli.Command { + return &ffcli.Command{ + Name: "version", + ShortUsage: "vidai version", + ShortHelp: "print version", + Exec: func(ctx context.Context, args []string) error { + v := version + if v == "" { + if buildInfo, ok := debug.ReadBuildInfo(); ok { + v = buildInfo.Main.Version + } + } + if v == "" { + v = "dev" + } + versionFields := []string{v} + if commit != "" { + versionFields = append(versionFields, commit) + } + if date != "" { + versionFields = append(versionFields, date) + } + fmt.Println(strings.Join(versionFields, " ")) + return nil + }, + } +} + +func newGenerateCommand() *ffcli.Command { + cmd := "generate" + fs := flag.NewFlagSet(cmd, flag.ExitOnError) + _ = fs.String("config", "", "config file (optional)") + + var cfg vidai.Config + fs.BoolVar(&cfg.Debug, "debug", false, "debug mode") + fs.DurationVar(&cfg.Wait, "wait", 2*time.Second, "wait time between requests") + fs.StringVar(&cfg.Token, "token", "", "runway token") + image := fs.String("image", "", "source image") + text := fs.String("text", "", "source text") + output := fs.String("output", "", "output file (optional, if omitted it won't be saved)") + extend := fs.Int("extend", 0, "extend the video by this many times (optional)") + interpolate := fs.Bool("interpolate", true, "interpolate frames (optional)") + upscale := fs.Bool("upscale", false, "upscale frames (optional)") + watermark := fs.Bool("watermark", false, "add watermark (optional)") + + return &ffcli.Command{ + Name: cmd, + ShortUsage: fmt.Sprintf("vidai %s [flags] ", cmd), + Options: []ff.Option{ + ff.WithConfigFileFlag("config"), + ff.WithConfigFileParser(ff.PlainParser), + ff.WithEnvVarPrefix("VIDAI"), + }, + ShortHelp: fmt.Sprintf("vidai %s command", cmd), + FlagSet: fs, + Exec: func(ctx context.Context, args []string) error { + if cfg.Token == "" { + return fmt.Errorf("token is required") + } + if *image == "" { + return fmt.Errorf("image is required") + } + c := vidai.New(&cfg) + urls, err := c.Generate(ctx, *image, *text, *output, *extend, + *interpolate, *upscale, *watermark) + if err != nil { + return err + } + if len(urls) == 1 { + fmt.Printf("Video URL: %s\n", urls[0]) + } else { + for i, u := range urls { + fmt.Printf("Video URL %d: %s\n", i+1, u) + } + } + return nil + }, + } +} + +func newExtendCommand() *ffcli.Command { + cmd := "extend" + fs := flag.NewFlagSet(cmd, flag.ExitOnError) + _ = fs.String("config", "", "config file (optional)") + + var cfg vidai.Config + fs.BoolVar(&cfg.Debug, "debug", false, "debug mode") + fs.DurationVar(&cfg.Wait, "wait", 2*time.Second, "wait time between requests") + fs.StringVar(&cfg.Token, "token", "", "runway token") + input := fs.String("input", "", "input video") + output := fs.String("output", "", "output file (optional, if omitted it won't be saved)") + n := fs.Int("n", 1, "extend the video by this many times") + interpolate := fs.Bool("interpolate", true, "interpolate frames (optional)") + upscale := fs.Bool("upscale", false, "upscale frames (optional)") + watermark := fs.Bool("watermark", false, "add watermark (optional)") + + return &ffcli.Command{ + Name: cmd, + ShortUsage: fmt.Sprintf("vidai %s [flags] ", cmd), + Options: []ff.Option{ + ff.WithConfigFileFlag("config"), + ff.WithConfigFileParser(ff.PlainParser), + ff.WithEnvVarPrefix("VIDAI"), + }, + ShortHelp: fmt.Sprintf("vidai %s command", cmd), + FlagSet: fs, + Exec: func(ctx context.Context, args []string) error { + if cfg.Token == "" { + return fmt.Errorf("token is required") + } + if *input == "" { + return fmt.Errorf("input is required") + } + if *n < 1 { + return fmt.Errorf("n must be greater than 0") + } + + c := vidai.New(&cfg) + u, err := c.Extend(ctx, *input, *output, *n, *interpolate, *upscale, *watermark) + if err != nil { + return err + } + fmt.Printf("Video URL: %s\n", u) + return nil + }, + } +} + +func newLoopCommand() *ffcli.Command { + cmd := "loop" + fs := flag.NewFlagSet(cmd, flag.ExitOnError) + _ = fs.String("config", "", "config file (optional)") + + input := fs.String("input", "", "input video") + output := fs.String("output", "", "output file") + + return &ffcli.Command{ + Name: cmd, + ShortUsage: fmt.Sprintf("vidai %s [flags] ", cmd), + Options: []ff.Option{ + ff.WithConfigFileFlag("config"), + ff.WithConfigFileParser(ff.PlainParser), + ff.WithEnvVarPrefix("VIDAI"), + }, + ShortHelp: fmt.Sprintf("vidai %s command", cmd), + FlagSet: fs, + Exec: func(ctx context.Context, args []string) error { + if *input == "" { + return fmt.Errorf("input is required") + } + if *output == "" { + return fmt.Errorf("output is required") + } + if err := vidai.Loop(ctx, *input, *output); err != nil { + return err + } + return nil + }, + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0f30c4c --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/igolaizola/vidai + +go 1.20 + +require github.com/peterbourgon/ff/v3 v3.3.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..bdc8b56 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/peterbourgon/ff/v3 v3.3.0 h1:PaKe7GW8orVFh8Unb5jNHS+JZBwWUMa2se0HM6/BI24= +github.com/peterbourgon/ff/v3 v3.3.0/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= diff --git a/internal/ratelimit/ratelimit.go b/internal/ratelimit/ratelimit.go new file mode 100644 index 0000000..970cc05 --- /dev/null +++ b/internal/ratelimit/ratelimit.go @@ -0,0 +1,47 @@ +package ratelimit + +import ( + "context" + "math/rand" + "sync" + "time" +) + +type Lock interface { + Lock(ctx context.Context) func() + LockWithDuration(ctx context.Context, d time.Duration) func() +} + +type lock struct { + lck *sync.Mutex + duration time.Duration +} + +// New creates a new rate limit lock. +func New(d time.Duration) Lock { + return &lock{ + lck: &sync.Mutex{}, + duration: d, + } +} + +// Lock locks the rate limit for the given duration and returns a function that +// unlocks the rate limit with a delay time based on the given duration. +func (l *lock) LockWithDuration(ctx context.Context, d time.Duration) func() { + l.lck.Lock() + // Apply a factor between 0.85 and 1.15 to the duration + d = time.Duration(float64(d) * (0.85 + rand.Float64()*0.3)) + return func() { + defer l.lck.Unlock() + select { + case <-ctx.Done(): + case <-time.After(d): + } + } +} + +// Lock locks the rate limit for the default duration and returns a function that +// unlocks the rate limit with a delay time based on the default duration. +func (l *lock) Lock(ctx context.Context) func() { + return l.LockWithDuration(ctx, l.duration) +} diff --git a/pkg/runway/runway.go b/pkg/runway/runway.go new file mode 100644 index 0000000..7a3f2f6 --- /dev/null +++ b/pkg/runway/runway.go @@ -0,0 +1,514 @@ +package runway + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math/rand" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/igolaizola/vidai/internal/ratelimit" +) + +type Client struct { + client *http.Client + debug bool + ratelimit ratelimit.Lock + token string + teamID int +} + +type Config struct { + Token string + Wait time.Duration + Debug bool + Client *http.Client +} + +func New(cfg *Config) *Client { + wait := cfg.Wait + if wait == 0 { + wait = 1 * time.Second + } + client := cfg.Client + if client == nil { + client = &http.Client{ + Timeout: 2 * time.Minute, + } + } + return &Client{ + client: client, + ratelimit: ratelimit.New(wait), + debug: cfg.Debug, + token: cfg.Token, + } +} + +type profileResponse struct { + User struct { + ID int `json:"id"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Email string `json:"email"` + Username string `json:"username"` + FirstName string `json:"firstName"` + LastName string `json:"lastName"` + IsVerified bool `json:"isVerified"` + GPUCredits int `json:"gpuCredits"` + GPUUsageLimit int `json:"gpuUsageLimit"` + Organizations []struct { + ID int `json:"id"` + Username string `json:"username"` + FirstName string `json:"firstName"` + LastName string `json:"lastName"` + Picture string `json:"picture"` + TeamName string `json:"teamName"` + TeamPicture string `json:"teamPicture"` + } `json:"organizations"` + } `json:"user"` +} + +func (c *Client) loadTeamID(ctx context.Context) error { + if c.teamID != 0 { + return nil + } + var resp profileResponse + if err := c.do(ctx, "GET", "profile", nil, &resp); err != nil { + return fmt.Errorf("runway: couldn't get profile: %w", err) + } + if len(resp.User.Organizations) > 0 { + c.teamID = resp.User.Organizations[0].ID + return nil + } + c.teamID = resp.User.ID + return nil +} + +type uploadRequest struct { + Filename string `json:"filename"` + NumberOfParts int `json:"numberOfParts"` + Type string `json:"type"` +} + +type uploadResponse struct { + ID string `json:"id"` + UploadURLs []string `json:"uploadUrls"` + UploadHeader map[string]string `json:"uploadHeaders"` +} + +type uploadCompleteRequest struct { + Parts []struct { + PartNumber int `json:"PartNumber"` + ETag string `json:"ETag"` + } `json:"parts"` +} + +type uploadCompleteResponse struct { + URL string `json:"url"` +} + +type uploadFile struct { + data []byte + extension string +} + +func (c *Client) Upload(ctx context.Context, name string, data []byte) (string, error) { + ext := strings.TrimPrefix(".", filepath.Ext(name)) + file := &uploadFile{ + data: data, + extension: ext, + } + + // Calculate etag + etag := fmt.Sprintf("%x", md5.Sum(data)) + + types := []string{ + "DATASET", + "DATASET_PREVIEW", + } + var imageURL string + for _, t := range types { + // Get upload URL + uploadReq := &uploadRequest{ + Filename: name, + NumberOfParts: 1, + Type: t, + } + var uploadResp uploadResponse + if err := c.do(ctx, "POST", "uploads", uploadReq, &uploadResp); err != nil { + return "", fmt.Errorf("runway: couldn't obtain upload url: %w", err) + } + if len(uploadResp.UploadURLs) == 0 { + return "", fmt.Errorf("runway: no upload urls returned") + } + + // Upload file + uploadURL := uploadResp.UploadURLs[0] + if err := c.do(ctx, "PUT", uploadURL, file, nil); err != nil { + return "", fmt.Errorf("runway: couldn't upload file: %w", err) + } + + // Complete upload + completeURL := fmt.Sprintf("uploads/%s/complete", uploadResp.ID) + completeReq := &uploadCompleteRequest{ + Parts: []struct { + PartNumber int `json:"PartNumber"` + ETag string `json:"ETag"` + }{ + { + PartNumber: 1, + ETag: etag, + }, + }, + } + var completeResp uploadCompleteResponse + if err := c.do(ctx, "POST", completeURL, completeReq, &completeResp); err != nil { + return "", fmt.Errorf("runway: couldn't complete upload: %w", err) + } + c.log("runway: upload complete %s", completeResp.URL) + if completeResp.URL == "" { + return "", fmt.Errorf("runway: empty image url for type %s", t) + } + imageURL = completeResp.URL + } + return imageURL, nil +} + +type createTaskRequest struct { + TaskType string `json:"taskType"` + Internal bool `json:"internal"` + Options struct { + Seconds int `json:"seconds"` + Gen2Options gen2Options `json:"gen2Options"` + Name string `json:"name"` + AssetGroupName string `json:"assetGroupName"` + ExploreMode bool `json:"exploreMode"` + } `json:"options"` + AsTeamID int `json:"asTeamId"` +} + +type gen2Options struct { + Interpolate bool `json:"interpolate"` + Seed int `json:"seed"` + Upscale bool `json:"upscale"` + TextPrompt string `json:"text_prompt"` + Watermark bool `json:"watermark"` + ImagePrompt string `json:"image_prompt"` + InitImage string `json:"init_image"` + Mode string `json:"mode"` +} + +type taskResponse struct { + Task struct { + ID string `json:"id"` + Name string `json:"name"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + TaskType string `json:"taskType"` + Options struct { + Seconds int `json:"seconds"` + Gen2Options gen2Options `json:"gen2Options"` + Name string `json:"name"` + AssetGroupName string `json:"assetGroupName"` + ExploreMode bool `json:"exploreMode"` + Recording bool `json:"recordingEnabled"` + } `json:"options"` + Status string `json:"status"` + ProgressText string `json:"progressText"` + ProgressRatio string `json:"progressRatio"` + PlaceInLine int `json:"placeInLine"` + EstimatedTimeToStartSeconds float64 `json:"estimatedTimeToStartSeconds"` + Artifacts []artifact `json:"artifacts"` + SharedAsset interface{} `json:"sharedAsset"` + } `json:"task"` +} + +type artifact struct { + ID string `json:"id"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + UserID int `json:"userId"` + CreatedBy int `json:"createdBy"` + TaskID string `json:"taskId"` + ParentAssetGroupId string `json:"parentAssetGroupId"` + Filename string `json:"filename"` + URL string `json:"url"` + FileSize int `json:"fileSize"` + IsDirectory bool `json:"isDirectory"` + PreviewURLs []string `json:"previewUrls"` + Private bool `json:"private"` + PrivateInTeam bool `json:"privateInTeam"` + Deleted bool `json:"deleted"` + Reported bool `json:"reported"` + Metadata struct { + FrameRate int `json:"frameRate"` + Duration int `json:"duration"` + Dimensions []int `json:"dimensions"` + } `json:"metadata"` +} + +func (c *Client) Generate(ctx context.Context, imageURL, textPrompt string, interpolate, upscale, watermark bool) (string, error) { + // Load team ID + if err := c.loadTeamID(ctx); err != nil { + return "", fmt.Errorf("runway: couldn't load team id: %w", err) + } + + // Generate seed + seed := rand.Intn(1000000000) + + // Create task + createReq := &createTaskRequest{ + TaskType: "gen2", + Internal: false, + Options: struct { + Seconds int `json:"seconds"` + Gen2Options gen2Options `json:"gen2Options"` + Name string `json:"name"` + AssetGroupName string `json:"assetGroupName"` + ExploreMode bool `json:"exploreMode"` + }{ + Seconds: 4, + Gen2Options: gen2Options{ + Interpolate: interpolate, + Seed: seed, + Upscale: upscale, + TextPrompt: textPrompt, + Watermark: watermark, + ImagePrompt: imageURL, + InitImage: imageURL, + Mode: "gen2", + }, + Name: fmt.Sprintf("Gen-2, %d", seed), + AssetGroupName: "Gen-2", + ExploreMode: false, + }, + AsTeamID: c.teamID, + } + var taskResp taskResponse + if err := c.do(ctx, "POST", "tasks", createReq, &taskResp); err != nil { + return "", fmt.Errorf("runway: couldn't create task: %w", err) + } + + // Wait for task to finish + for { + switch taskResp.Task.Status { + case "SUCCEEDED": + if len(taskResp.Task.Artifacts) == 0 { + return "", fmt.Errorf("runway: no artifacts returned") + } + if taskResp.Task.Artifacts[0].URL == "" { + return "", fmt.Errorf("runway: empty artifact url") + } + return taskResp.Task.Artifacts[0].URL, nil + case "PENDING", "RUNNING": + c.log("runway: task %s: %s", taskResp.Task.ID, taskResp.Task.ProgressRatio) + default: + return "", fmt.Errorf("runway: task failed: %s", taskResp.Task.Status) + } + + select { + case <-ctx.Done(): + return "", fmt.Errorf("runway: %w", ctx.Err()) + case <-time.After(5 * time.Second): + } + + path := fmt.Sprintf("tasks/%s?asTeamId=%d", taskResp.Task.ID, c.teamID) + if err := c.do(ctx, "GET", path, nil, &taskResp); err != nil { + return "", fmt.Errorf("runway: couldn't get task: %w", err) + } + } +} + +type assetDeleteRequest struct { +} + +type assetDeleteResponse struct { + Success bool `json:"success"` +} + +// TODO: Delete asset by url instead +func (c *Client) DeleteAsset(ctx context.Context, id string) error { + path := fmt.Sprintf("assets/%s", id) + var resp assetDeleteResponse + if err := c.do(ctx, "DELETE", path, &assetDeleteRequest{}, &resp); err != nil { + return fmt.Errorf("runway: couldn't delete asset %s: %w", id, err) + } + if !resp.Success { + return fmt.Errorf("runway: couldn't delete asset %s", id) + } + return nil +} + +func (c *Client) log(format string, args ...interface{}) { + if c.debug { + format += "\n" + log.Printf(format, args...) + } +} + +var backoff = []time.Duration{ + 30 * time.Second, + 1 * time.Minute, + 2 * time.Minute, +} + +func (c *Client) do(ctx context.Context, method, path string, in, out any) error { + maxAttempts := 3 + attempts := 0 + var err error + for { + if err != nil { + log.Println("retrying...", err) + } + err = c.doAttempt(ctx, method, path, in, out) + if err == nil { + return nil + } + // Increase attempts and check if we should stop + attempts++ + if attempts >= maxAttempts { + return err + } + // If the error is temporary retry + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + continue + } + // Check status code + var errStatus errStatusCode + if errors.As(err, &errStatus) { + switch int(errStatus) { + // These errors are retriable but we should wait before retry + case http.StatusBadGateway, http.StatusGatewayTimeout, http.StatusTooManyRequests: + default: + return err + } + + idx := attempts - 1 + if idx >= len(backoff) { + idx = len(backoff) - 1 + } + wait := backoff[idx] + c.log("server seems to be down, waiting %s before retrying\n", wait) + t := time.NewTimer(wait) + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + } + continue + } + return err + } +} + +type errStatusCode int + +func (e errStatusCode) Error() string { + return fmt.Sprintf("%d", e) +} + +func (c *Client) doAttempt(ctx context.Context, method, path string, in, out any) error { + var body []byte + var reqBody io.Reader + contentType := "application/json" + if f, ok := in.(*uploadFile); ok { + body = f.data + ext := f.extension + if ext == "jpg" { + ext = "jpeg" + } + contentType = fmt.Sprintf("image/%s", ext) + reqBody = bytes.NewReader(body) + } else if in != nil { + var err error + body, err = json.Marshal(in) + if err != nil { + return fmt.Errorf("runway: couldn't marshal request body: %w", err) + } + reqBody = bytes.NewReader(body) + } + logBody := string(body) + if len(logBody) > 100 { + logBody = logBody[:100] + "..." + } + c.log("runway: do %s %s %s", method, path, logBody) + + // Check if path is absolute + u := fmt.Sprintf("https://api.runwayml.com/v1/%s", path) + var uploadLen int + if strings.HasPrefix(path, "http") { + u = path + uploadLen = len(body) + } + req, err := http.NewRequestWithContext(ctx, method, u, reqBody) + if err != nil { + return fmt.Errorf("runway: couldn't create request: %w", err) + } + c.addHeaders(req, contentType, uploadLen) + + unlock := c.ratelimit.Lock(ctx) + defer unlock() + + resp, err := c.client.Do(req) + if err != nil { + return fmt.Errorf("runway: couldn't %s %s: %w", method, u, err) + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("runway: couldn't read response body: %w", err) + } + c.log("runway: response %s %s %d %s", method, path, resp.StatusCode, string(respBody)) + if resp.StatusCode != http.StatusOK { + errMessage := string(respBody) + if len(errMessage) > 100 { + errMessage = errMessage[:100] + "..." + } + _ = os.WriteFile(fmt.Sprintf("logs/debug_%s.json", time.Now().Format("20060102_150405")), respBody, 0644) + return fmt.Errorf("runway: %s %s returned (%s): %w", method, u, errMessage, errStatusCode(resp.StatusCode)) + } + if out != nil { + if err := json.Unmarshal(respBody, out); err != nil { + // Write response body to file for debugging. + _ = os.WriteFile(fmt.Sprintf("logs/debug_%s.json", time.Now().Format("20060102_150405")), respBody, 0644) + return fmt.Errorf("runway: couldn't unmarshal response body (%T): %w", out, err) + } + } + return nil +} + +func (c *Client) addHeaders(req *http.Request, contentType string, uploadLen int) { + if uploadLen > 0 { + req.Header.Set("Accept", "*/*") + req.Header.Set("Content-Length", fmt.Sprintf("%d", uploadLen)) + req.Header.Set("Sec-Fetch-Site", "cross-site") + } else { + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.token)) + req.Header.Set("Sec-Fetch-Site", "same-site") + } + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Connection", "keep-alive") + req.Header.Set("Content-Type", contentType) + req.Header.Set("Origin", "https://app.runwayml.com") + req.Header.Set("Referer", "https://app.runwayml.com/") + req.Header.Set("Sec-Ch-Ua", `"Not.A/Brand";v="8", "Chromium";v="114", "Microsoft Edge";v="114"`) + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", "\"Windows\"") + req.Header.Set("Sec-Fetch-Dest", "empty") + req.Header.Set("Sec-Fetch-Mode", "cors") + // TODO: Add sentry trace if needed. + // req.Header.Set("Sentry-Trace", "TODO") + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.82") +} diff --git a/vidai.go b/vidai.go new file mode 100644 index 0000000..535a295 --- /dev/null +++ b/vidai.go @@ -0,0 +1,263 @@ +package vidai + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/igolaizola/vidai/pkg/runway" +) + +type Client struct { + client *runway.Client + httpClient *http.Client +} + +type Config struct { + Token string + Wait time.Duration + Debug bool + Client *http.Client +} + +func New(cfg *Config) *Client { + httpClient := cfg.Client + if httpClient == nil { + httpClient = &http.Client{ + Timeout: 2 * time.Minute, + } + } + client := runway.New(&runway.Config{ + Token: cfg.Token, + Wait: cfg.Wait, + Debug: cfg.Debug, + Client: httpClient, + }) + return &Client{ + client: client, + httpClient: httpClient, + } +} + +// Generate generates a video from an image and a text prompt. +func (c *Client) Generate(ctx context.Context, image, text, output string, + extend int, interpolate, upscale, watermark bool) ([]string, error) { + b, err := os.ReadFile(image) + if err != nil { + return nil, fmt.Errorf("vidai: couldn't read image: %w", err) + } + name := filepath.Base(image) + + imageURL, err := c.client.Upload(ctx, name, b) + if err != nil { + return nil, fmt.Errorf("vidai: couldn't upload image: %w", err) + } + videoURL, err := c.client.Generate(ctx, imageURL, text, interpolate, upscale, watermark) + if err != nil { + return nil, fmt.Errorf("vidai: couldn't generate video: %w", err) + } + + // Use temp file if no output is set and we need to extend the video + videoPath := output + if videoPath == "" && extend > 0 { + base := strings.TrimSuffix(filepath.Base(image), filepath.Ext(image)) + videoPath = filepath.Join(os.TempDir(), fmt.Sprintf("%s.mp4", base)) + } + + // Download video + if videoPath != "" { + if err := c.download(ctx, videoURL, videoPath); err != nil { + return nil, fmt.Errorf("vidai: couldn't download video: %w", err) + } + } + + // Extend video + if extend > 0 { + extendURLs, err := c.Extend(ctx, videoPath, output, extend, + interpolate, upscale, watermark) + if err != nil { + return nil, fmt.Errorf("vidai: couldn't extend video: %w", err) + } + return append([]string{output}, extendURLs...), nil + } + + return []string{videoURL}, nil +} + +// Extend extends a video using the last frame of the previous video. +func (c *Client) Extend(ctx context.Context, input, output string, n int, + interpolate, upscale, watermark bool) ([]string, error) { + base := strings.TrimSuffix(filepath.Base(input), filepath.Ext(input)) + + // Copy input video to temp file + vid := filepath.Join(os.TempDir(), fmt.Sprintf("%s-0.mp4", base)) + if err := copyFile(input, vid); err != nil { + return nil, fmt.Errorf("vidai: couldn't copy input video: %w", err) + } + + videos := []string{vid} + var urls []string + for i := 0; i < n; i++ { + img := filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d.jpg", base, i)) + + // Extract last frame from video using the following command: + // ffmpeg -sseof -1 -i input.mp4 -update 1 -q:v 1 output.jpg + // This will seek to the last second of the input and output all frames. + // But since -update 1 is set, each frame will be overwritten to the + // same file, leaving only the last frame remaining. + cmd := exec.CommandContext(ctx, "ffmpeg", "-sseof", "-1", "-i", vid, "-update", "1", "-q:v", "1", img) + cmdOut, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("vidai: couldn't extract last frame (%s): %w", string(cmdOut), err) + } + + // Read image + b, err := os.ReadFile(img) + if err != nil { + return nil, fmt.Errorf("vidai: couldn't read image: %w", err) + } + name := filepath.Base(img) + + // Generate video + imageURL, err := c.client.Upload(ctx, name, b) + if err != nil { + return nil, fmt.Errorf("vidai: couldn't upload image: %w", err) + } + videoURL, err := c.client.Generate(ctx, imageURL, "", interpolate, upscale, watermark) + if err != nil { + return nil, fmt.Errorf("vidai: couldn't generate video: %w", err) + } + + // Remove temporary image + if err := os.Remove(img); err != nil { + log.Println(fmt.Errorf("vidai: couldn't remove image: %w", err)) + } + + // Download video to temp file + vid = filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d.mp4", base, i+1)) + if err := c.download(ctx, videoURL, vid); err != nil { + return nil, fmt.Errorf("vidai: couldn't download video: %w", err) + } + videos = append(videos, vid) + } + + if output != "" { + // Create list of videos + var listData string + for _, v := range videos { + listData += fmt.Sprintf("file '%s'\n", filepath.Base(v)) + } + list := filepath.Join(os.TempDir(), fmt.Sprintf("%s-list.txt", base)) + if err := os.WriteFile(list, []byte(listData), 0644); err != nil { + return nil, fmt.Errorf("vidai: couldn't create list file: %w", err) + } + + // Combine videos using the following command: + // ffmpeg -f concat -safe 0 -i list.txt -c copy output.mp4 + cmd := exec.CommandContext(ctx, "ffmpeg", "-f", "concat", "-safe", "0", "-i", list, "-c", "copy", "-y", output) + cmdOut, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("vidai: couldn't combine videos (%s): %w", string(cmdOut), err) + } + + // Remove temporary list file + if err := os.Remove(list); err != nil { + log.Println(fmt.Errorf("vidai: couldn't remove list file: %w", err)) + } + } + + // Remove temporary videos + for _, v := range videos { + if err := os.Remove(v); err != nil { + log.Println(fmt.Errorf("vidai: couldn't remove video: %w", err)) + } + } + + return urls, nil +} + +func (c *Client) download(ctx context.Context, url, output string) error { + // Create request + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("vidai: couldn't create request: %w", err) + } + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("vidai: couldn't download video: %w", err) + } + defer resp.Body.Close() + + // Write video to output + f, err := os.Create(output) + if err != nil { + return fmt.Errorf("vidai: couldn't create temp file: %w", err) + } + defer f.Close() + if _, err := io.Copy(f, resp.Body); err != nil { + return fmt.Errorf("vidai: couldn't write to temp file: %w", err) + } + return nil +} + +func copyFile(src, dst string) error { + // Open source file + srcFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("vidai: couldn't open source file: %w", err) + } + defer srcFile.Close() + + // Create destination file + dstFile, err := os.Create(dst) + if err != nil { + return fmt.Errorf("vidai: couldn't create destination file: %w", err) + } + defer dstFile.Close() + + // Copy source to destination + if _, err := io.Copy(dstFile, srcFile); err != nil { + return fmt.Errorf("vidai: couldn't copy source to destination: %w", err) + } + return nil +} + +func Loop(ctx context.Context, input, output string) error { + // Reverse video using the following command: + // ffmpeg -i input.mp4 -vf reverse temp.mp4 + tmp := filepath.Join(os.TempDir(), fmt.Sprintf("%s-reversed.mp4", filepath.Base(input))) + cmd := exec.CommandContext(ctx, "ffmpeg", "-i", input, "-vf", "reverse", tmp) + cmdOut, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("vidai: couldn't reverse video (%s): %w", string(cmdOut), err) + } + + // Obtain absolute path to input video + absInput, err := filepath.Abs(input) + if err != nil { + return fmt.Errorf("vidai: couldn't get absolute path to input video: %w", err) + } + + // Generate list of videos + listData := fmt.Sprintf("file '%s'\nfile '%s'\n", absInput, filepath.Base(tmp)) + list := filepath.Join(os.TempDir(), fmt.Sprintf("%s-list.txt", filepath.Base(input))) + if err := os.WriteFile(list, []byte(listData), 0644); err != nil { + return fmt.Errorf("vidai: couldn't create list file: %w", err) + } + + // Combine videos using the following command: + // ffmpeg -f concat -safe 0 -i list.txt -c copy output.mp4 + cmd = exec.CommandContext(ctx, "ffmpeg", "-f", "concat", "-safe", "0", "-i", list, "-c", "copy", "-y", output) + cmdOut, err = cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("vidai: couldn't combine videos (%s): %w", string(cmdOut), err) + } + return nil +}