feat: Support non-embedded python via the same interface

.github/workflows/release.yml

@@ -11,13 +11,13 @@ on:
 env:
   PYTHON_STANDALONE_VERSIONS: |
     [
-      "20240224"
+      "20240415"
     ]
   PYTHON_VERSIONS: |
     [
-      "3.10.13",
-      "3.11.8",
-      "3.12.2"
+      "3.10.14",
+      "3.11.9",
+      "3.12.3"
     ]
 
 jobs:
@@ -39,14 +39,24 @@ jobs:
         pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
         pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
       fail-fast: false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - name: checkout
+      - name: clone
         run: |
           # can't use actions/checkout here as transferring the shallow clone fails when using upload-/download-artifact
-          git clone https://token:[email protected]/$GITHUB_REPOSITORY . --depth=1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          git clone https://github.com/$GITHUB_REPOSITORY . --depth=1
+      - name: checkout PR
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          echo fetching pull/${{ github.ref_name }}
+          git fetch origin pull/${{ github.ref_name }}:pr --depth=1
+          git checkout pr
+      - name: checkout branch
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          echo fetching ${{ github.ref_name }}
+          git fetch origin ${{ github.ref_name }} --depth=1
+          git checkout ${{ github.ref_name }}
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
@@ -83,8 +93,8 @@ jobs:
     strategy:
       matrix:
         os:
-          - ubuntu-20.04
-          - macos-11
+          - ubuntu-22.04
+          - macos-12
           - windows-2019
         pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
         pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
@@ -125,7 +135,7 @@ jobs:
         pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }}
         pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }}
       fail-fast: false
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
     permissions:
       contents: write

example/main.go

@@ -11,7 +11,10 @@ func main() {
 		panic(err)
 	}
 
-	cmd := ep.PythonCmd("-c", "print('hello')")
+	cmd, err := ep.PythonCmd("-c", "print('hello')")
+	if err != nil {
+		panic(err)
+	}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()

go.mod

@@ -4,7 +4,7 @@ go 1.19
 
 require (
 	github.com/gobwas/glob v0.2.3
-	github.com/klauspost/compress v1.17.8
+	github.com/klauspost/compress v1.17.9
 	github.com/rogpeppe/go-internal v1.12.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
@@ -14,6 +14,6 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -7,6 +7,8 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA
 github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
 github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -24,6 +26,8 @@ golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

hack/build-tag.sh

@@ -35,7 +35,7 @@ go run ./pip/generate
 TAG=v0.0.0-$PYTHON_VERSION-$PYTHON_STANDALONE_VERSION-$BUILD_NUM
 
 echo "checking out temporary branch"
-git checkout $(git rev-parse HEAD)
+git checkout --detach
 git add -f python/internal/data
 git add -f pip/internal/data
 git commit -m "added python $PYTHON_VERSION from python-standalone $PYTHON_STANDALONE_VERSION"

pip/embed_pip_packages.go

@@ -109,10 +109,13 @@ func pipInstall(ep *python.EmbeddedPython, requirementsFile string, platforms []
 		args = append(args, "--only-binary=:all:")
 	}
 
-	cmd := ep.PythonCmd(args...)
+	cmd, err := ep.PythonCmd(args...)
+	if err != nil {
+		return err
+	}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	err := cmd.Run()
+	err = cmd.Run()
 	if err != nil {
 		return err
 	}

pip/generate/main.go

@@ -33,10 +33,13 @@ func bootstrapPip(ep *python.EmbeddedPython) {
 	getPip := downloadGetPip()
 	defer os.Remove(getPip)
 
-	cmd := ep.PythonCmd(getPip)
+	cmd, err := ep.PythonCmd(getPip)
+	if err != nil {
+		panic(err)
+	}
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
-	err := cmd.Run()
+	err = cmd.Run()
 	if err != nil {
 		panic(err)
 	}

python/embedded_python.go

@@ -4,17 +4,11 @@ import (
 	"fmt"
 	"github.com/kluctl/go-embed-python/embed_util"
 	"github.com/kluctl/go-embed-python/python/internal/data"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strings"
 )
 
 type EmbeddedPython struct {
 	e *embed_util.EmbeddedFiles
-
-	pythonPath []string
+	*Python
 }
 
 // NewEmbeddedPython creates a new EmbeddedPython instance. The embedded source code and python binaries are
@@ -26,7 +20,8 @@ func NewEmbeddedPython(name string) (*EmbeddedPython, error) {
 		return nil, err
 	}
 	return &EmbeddedPython{
-		e: e,
+		e:      e,
+		Python: NewPython(WithPythonHome(e.GetExtractedPath())),
 	}, nil
 }
 
@@ -36,7 +31,8 @@ func NewEmbeddedPythonWithTmpDir(tmpDir string, withHashInDir bool) (*EmbeddedPy
 		return nil, err
 	}
 	return &EmbeddedPython{
-		e: e,
+		e:      e,
+		Python: NewPython(WithPythonHome(e.GetExtractedPath())),
 	}, nil
 }
 
@@ -47,44 +43,3 @@ func (ep *EmbeddedPython) Cleanup() error {
 func (ep *EmbeddedPython) GetExtractedPath() string {
 	return ep.e.GetExtractedPath()
 }
-
-func (ep *EmbeddedPython) GetBinPath() string {
-	if runtime.GOOS == "windows" {
-		return ep.GetExtractedPath()
-	} else {
-		return filepath.Join(ep.GetExtractedPath(), "bin")
-	}
-}
-
-func (ep *EmbeddedPython) GetExePath() string {
-	suffix := ""
-	if runtime.GOOS == "windows" {
-		suffix = ".exe"
-	} else {
-		suffix = "3"
-	}
-	return filepath.Join(ep.GetBinPath(), "python"+suffix)
-}
-
-func (ep *EmbeddedPython) AddPythonPath(p string) {
-	ep.pythonPath = append(ep.pythonPath, p)
-}
-
-func (ep *EmbeddedPython) PythonCmd(args ...string) *exec.Cmd {
-	return ep.PythonCmd2(args)
-}
-
-func (ep *EmbeddedPython) PythonCmd2(args []string) *exec.Cmd {
-	exePath := ep.GetExePath()
-
-	cmd := exec.Command(exePath, args...)
-	cmd.Env = os.Environ()
-	cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONHOME=%s", ep.GetExtractedPath()))
-
-	if len(ep.pythonPath) != 0 {
-		pythonPathEnv := fmt.Sprintf("PYTHONPATH=%s", strings.Join(ep.pythonPath, string(os.PathListSeparator)))
-		cmd.Env = append(cmd.Env, pythonPathEnv)
-	}
-
-	return cmd
-}

python/embedded_python_test.go

@@ -17,10 +17,10 @@ func TestEmbeddedPython(t *testing.T) {
 	defer ep.Cleanup()
 	path := ep.GetExtractedPath()
 	assert.NotEqual(t, path, "")
-	pexe := ep.GetExePath()
+	pexe, _ := ep.GetExePath()
 	assert.True(t, internal.Exists(pexe))
 
-	cmd := ep.PythonCmd("-c", "print('test test')")
+	cmd, _ := ep.PythonCmd("-c", "print('test test')")
 	stdout, err := cmd.StdoutPipe()
 	assert.NoError(t, err)
 	defer stdout.Close()
@@ -61,10 +61,10 @@ print("platform.processor=" + platform.processor())
 	defer ep.Cleanup()
 	path := ep.GetExtractedPath()
 	assert.NotEqual(t, path, "")
-	pexe := ep.GetExePath()
+	pexe, _ := ep.GetExePath()
 	assert.True(t, internal.Exists(pexe))
 
-	cmd := ep.PythonCmd("-c", getSystemInfo)
+	cmd, _ := ep.PythonCmd("-c", getSystemInfo)
 	stdout, _ := cmd.StdoutPipe()
 	stderr, _ := cmd.StderrPipe()
 	defer stdout.Close()

python/generate/main.go

@@ -81,10 +81,10 @@ func main() {
 	}
 
 	jobs := []job{
-		{"linux", "amd64", "unknown-linux-gnu-lto-full", keepNixPatterns},
+		{"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns},
 		{"linux", "arm64", "unknown-linux-gnu-lto-full", keepNixPatterns},
-		{"darwin", "amd64", "apple-darwin-lto-full", keepNixPatterns},
-		{"darwin", "arm64", "apple-darwin-lto-full", keepNixPatterns},
+		{"darwin", "amd64", "apple-darwin-pgo+lto-full", keepNixPatterns},
+		{"darwin", "arm64", "apple-darwin-pgo+lto-full", keepNixPatterns},
 		{"windows", "amd64", "pc-windows-msvc-shared-pgo-full", keepWinPatterns},
 	}
 	for _, j := range jobs {

python/python.go

@@ -0,0 +1,93 @@
+package python
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type Python struct {
+	pythonHome string
+	pythonPath []string
+}
+
+type PythonOpt func(o *Python)
+
+func WithPythonHome(home string) PythonOpt {
+	return func(o *Python) {
+		o.pythonHome = home
+	}
+}
+
+func NewPython(opts ...PythonOpt) *Python {
+	ep := &Python{}
+
+	for _, o := range opts {
+		o(ep)
+	}
+
+	return ep
+}
+
+func (ep *Python) GetExeName() string {
+	suffix := ""
+	if runtime.GOOS == "windows" {
+		suffix = ".exe"
+	} else {
+		suffix = "3"
+	}
+	return "python" + suffix
+}
+
+func (ep *Python) GetExePath() (string, error) {
+	if ep.pythonHome == "" {
+		p, err := exec.LookPath(ep.GetExeName())
+		if err != nil {
+			return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err)
+		}
+		return p, nil
+	} else {
+		var p string
+		if runtime.GOOS == "windows" {
+			p = filepath.Join(ep.pythonHome, ep.GetExeName())
+		} else {
+			p = filepath.Join(ep.pythonHome, "bin", ep.GetExeName())
+		}
+		if _, err := os.Stat(p); err != nil {
+			return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err)
+		}
+		return p, nil
+	}
+}
+
+func (ep *Python) AddPythonPath(p string) {
+	ep.pythonPath = append(ep.pythonPath, p)
+}
+
+func (ep *Python) PythonCmd(args ...string) (*exec.Cmd, error) {
+	return ep.PythonCmd2(args)
+}
+
+func (ep *Python) PythonCmd2(args []string) (*exec.Cmd, error) {
+	exePath, err := ep.GetExePath()
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := exec.Command(exePath, args...)
+	cmd.Env = os.Environ()
+
+	if ep.pythonHome != "" {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONHOME=%s", ep.pythonHome))
+	}
+
+	if len(ep.pythonPath) != 0 {
+		pythonPathEnv := fmt.Sprintf("PYTHONPATH=%s", strings.Join(ep.pythonPath, string(os.PathListSeparator)))
+		cmd.Env = append(cmd.Env, pythonPathEnv)
+	}
+
+	return cmd, nil
+}