Skip to content

Commit

Permalink
Merge pull request #43 from kluctl/feat-external-python
Browse files Browse the repository at this point in the history
feat: Support non-embedded python via the same interface
  • Loading branch information
codablock authored Jun 17, 2024
2 parents 1ba1472 + 421798e commit b1e2e38
Show file tree
Hide file tree
Showing 12 changed files with 181 additions and 77 deletions.
34 changes: 22 additions & 12 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@ on:
env:
PYTHON_STANDALONE_VERSIONS: |
[
"20240224"
"20240415"
]
PYTHON_VERSIONS: |
[
"3.10.13",
"3.11.8",
"3.12.2"
"3.10.14",
"3.11.9",
"3.12.3"
]
jobs:
Expand All @@ -39,14 +39,24 @@ jobs:
pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
fail-fast: false
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
steps:
- name: checkout
- name: clone
run: |
# can't use actions/checkout here as transferring the shallow clone fails when using upload-/download-artifact
git clone https://token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY . --depth=1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
git clone https://github.com/$GITHUB_REPOSITORY . --depth=1
- name: checkout PR
if: ${{ github.event_name == 'pull_request' }}
run: |
echo fetching pull/${{ github.ref_name }}
git fetch origin pull/${{ github.ref_name }}:pr --depth=1
git checkout pr
- name: checkout branch
if: ${{ github.event_name == 'push' }}
run: |
echo fetching ${{ github.ref_name }}
git fetch origin ${{ github.ref_name }} --depth=1
git checkout ${{ github.ref_name }}
- name: Set up Go
uses: actions/setup-go@v5
with:
Expand Down Expand Up @@ -83,8 +93,8 @@ jobs:
strategy:
matrix:
os:
- ubuntu-20.04
- macos-11
- ubuntu-22.04
- macos-12
- windows-2019
pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
Expand Down Expand Up @@ -125,7 +135,7 @@ jobs:
pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
fail-fast: false
runs-on: ubuntu-20.04
runs-on: ubuntu-22.04
if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
permissions:
contents: write
Expand Down
5 changes: 4 additions & 1 deletion example/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ func main() {
panic(err)
}

cmd := ep.PythonCmd("-c", "print('hello')")
cmd, err := ep.PythonCmd("-c", "print('hello')")
if err != nil {
panic(err)
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err = cmd.Run()
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.19

require (
github.com/gobwas/glob v0.2.3
github.com/klauspost/compress v1.17.8
github.com/klauspost/compress v1.17.9
github.com/rogpeppe/go-internal v1.12.0
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
Expand All @@ -14,6 +14,6 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/sys v0.21.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
Expand All @@ -24,6 +26,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
Expand Down
2 changes: 1 addition & 1 deletion hack/build-tag.sh
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ go run ./pip/generate
TAG=v0.0.0-$PYTHON_VERSION-$PYTHON_STANDALONE_VERSION-$BUILD_NUM

echo "checking out temporary branch"
git checkout $(git rev-parse HEAD)
git checkout --detach
git add -f python/internal/data
git add -f pip/internal/data
git commit -m "added python $PYTHON_VERSION from python-standalone $PYTHON_STANDALONE_VERSION"
Expand Down
7 changes: 5 additions & 2 deletions pip/embed_pip_packages.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,13 @@ func pipInstall(ep *python.EmbeddedPython, requirementsFile string, platforms []
args = append(args, "--only-binary=:all:")
}

cmd := ep.PythonCmd(args...)
cmd, err := ep.PythonCmd(args...)
if err != nil {
return err
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
err = cmd.Run()
if err != nil {
return err
}
Expand Down
7 changes: 5 additions & 2 deletions pip/generate/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,13 @@ func bootstrapPip(ep *python.EmbeddedPython) {
getPip := downloadGetPip()
defer os.Remove(getPip)

cmd := ep.PythonCmd(getPip)
cmd, err := ep.PythonCmd(getPip)
if err != nil {
panic(err)
}
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
err = cmd.Run()
if err != nil {
panic(err)
}
Expand Down
55 changes: 5 additions & 50 deletions python/embedded_python.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,11 @@ import (
"fmt"
"github.com/kluctl/go-embed-python/embed_util"
"github.com/kluctl/go-embed-python/python/internal/data"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
)

type EmbeddedPython struct {
e *embed_util.EmbeddedFiles

pythonPath []string
*Python
}

// NewEmbeddedPython creates a new EmbeddedPython instance. The embedded source code and python binaries are
Expand All @@ -26,7 +20,8 @@ func NewEmbeddedPython(name string) (*EmbeddedPython, error) {
return nil, err
}
return &EmbeddedPython{
e: e,
e: e,
Python: NewPython(WithPythonHome(e.GetExtractedPath())),
}, nil
}

Expand All @@ -36,7 +31,8 @@ func NewEmbeddedPythonWithTmpDir(tmpDir string, withHashInDir bool) (*EmbeddedPy
return nil, err
}
return &EmbeddedPython{
e: e,
e: e,
Python: NewPython(WithPythonHome(e.GetExtractedPath())),
}, nil
}

Expand All @@ -47,44 +43,3 @@ func (ep *EmbeddedPython) Cleanup() error {
func (ep *EmbeddedPython) GetExtractedPath() string {
return ep.e.GetExtractedPath()
}

func (ep *EmbeddedPython) GetBinPath() string {
if runtime.GOOS == "windows" {
return ep.GetExtractedPath()
} else {
return filepath.Join(ep.GetExtractedPath(), "bin")
}
}

func (ep *EmbeddedPython) GetExePath() string {
suffix := ""
if runtime.GOOS == "windows" {
suffix = ".exe"
} else {
suffix = "3"
}
return filepath.Join(ep.GetBinPath(), "python"+suffix)
}

func (ep *EmbeddedPython) AddPythonPath(p string) {
ep.pythonPath = append(ep.pythonPath, p)
}

func (ep *EmbeddedPython) PythonCmd(args ...string) *exec.Cmd {
return ep.PythonCmd2(args)
}

func (ep *EmbeddedPython) PythonCmd2(args []string) *exec.Cmd {
exePath := ep.GetExePath()

cmd := exec.Command(exePath, args...)
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONHOME=%s", ep.GetExtractedPath()))

if len(ep.pythonPath) != 0 {
pythonPathEnv := fmt.Sprintf("PYTHONPATH=%s", strings.Join(ep.pythonPath, string(os.PathListSeparator)))
cmd.Env = append(cmd.Env, pythonPathEnv)
}

return cmd
}
8 changes: 4 additions & 4 deletions python/embedded_python_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ func TestEmbeddedPython(t *testing.T) {
defer ep.Cleanup()
path := ep.GetExtractedPath()
assert.NotEqual(t, path, "")
pexe := ep.GetExePath()
pexe, _ := ep.GetExePath()
assert.True(t, internal.Exists(pexe))

cmd := ep.PythonCmd("-c", "print('test test')")
cmd, _ := ep.PythonCmd("-c", "print('test test')")
stdout, err := cmd.StdoutPipe()
assert.NoError(t, err)
defer stdout.Close()
Expand Down Expand Up @@ -61,10 +61,10 @@ print("platform.processor=" + platform.processor())
defer ep.Cleanup()
path := ep.GetExtractedPath()
assert.NotEqual(t, path, "")
pexe := ep.GetExePath()
pexe, _ := ep.GetExePath()
assert.True(t, internal.Exists(pexe))

cmd := ep.PythonCmd("-c", getSystemInfo)
cmd, _ := ep.PythonCmd("-c", getSystemInfo)
stdout, _ := cmd.StdoutPipe()
stderr, _ := cmd.StderrPipe()
defer stdout.Close()
Expand Down
6 changes: 3 additions & 3 deletions python/generate/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,10 +81,10 @@ func main() {
}

jobs := []job{
{"linux", "amd64", "unknown-linux-gnu-lto-full", keepNixPatterns},
{"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns},
{"linux", "arm64", "unknown-linux-gnu-lto-full", keepNixPatterns},
{"darwin", "amd64", "apple-darwin-lto-full", keepNixPatterns},
{"darwin", "arm64", "apple-darwin-lto-full", keepNixPatterns},
{"darwin", "amd64", "apple-darwin-pgo+lto-full", keepNixPatterns},
{"darwin", "arm64", "apple-darwin-pgo+lto-full", keepNixPatterns},
{"windows", "amd64", "pc-windows-msvc-shared-pgo-full", keepWinPatterns},
}
for _, j := range jobs {
Expand Down
93 changes: 93 additions & 0 deletions python/python.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package python

import (
"fmt"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
)

type Python struct {
pythonHome string
pythonPath []string
}

type PythonOpt func(o *Python)

func WithPythonHome(home string) PythonOpt {
return func(o *Python) {
o.pythonHome = home
}
}

func NewPython(opts ...PythonOpt) *Python {
ep := &Python{}

for _, o := range opts {
o(ep)
}

return ep
}

func (ep *Python) GetExeName() string {
suffix := ""
if runtime.GOOS == "windows" {
suffix = ".exe"
} else {
suffix = "3"
}
return "python" + suffix
}

func (ep *Python) GetExePath() (string, error) {
if ep.pythonHome == "" {
p, err := exec.LookPath(ep.GetExeName())
if err != nil {
return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err)
}
return p, nil
} else {
var p string
if runtime.GOOS == "windows" {
p = filepath.Join(ep.pythonHome, ep.GetExeName())
} else {
p = filepath.Join(ep.pythonHome, "bin", ep.GetExeName())
}
if _, err := os.Stat(p); err != nil {
return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err)
}
return p, nil
}
}

func (ep *Python) AddPythonPath(p string) {
ep.pythonPath = append(ep.pythonPath, p)
}

func (ep *Python) PythonCmd(args ...string) (*exec.Cmd, error) {
return ep.PythonCmd2(args)
}

func (ep *Python) PythonCmd2(args []string) (*exec.Cmd, error) {
exePath, err := ep.GetExePath()
if err != nil {
return nil, err
}

cmd := exec.Command(exePath, args...)
cmd.Env = os.Environ()

if ep.pythonHome != "" {
cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONHOME=%s", ep.pythonHome))
}

if len(ep.pythonPath) != 0 {
pythonPathEnv := fmt.Sprintf("PYTHONPATH=%s", strings.Join(ep.pythonPath, string(os.PathListSeparator)))
cmd.Env = append(cmd.Env, pythonPathEnv)
}

return cmd, nil
}
Loading

0 comments on commit b1e2e38

Please sign in to comment.