Files
cpy3/lifecycle.go
zeekayandGitHub aa54594e88 Hanzo fork v0.2.1: Python 3.12+ compat, sub-interp wrapper, free-threading detection, JSON hot-path + 3.13t build script (#1)
* feat: Python 3.12+/3.13 compat + sub-interpreter wrapper

Hanzo fork of go-python/cpy3. See HANZO_CHANGES.md for full diff.

- Polyfill APIs removed in 3.12 (per-type ClearFreeList helpers) and
  3.13 (PyEval_InitThreads, Py_SetPath/PythonHome/ProgramName,
  PySys_SetArgv/AddWarnOption/AddXOption/SetPath,
  Py_SetStandardStreamEncoding, PyEval_ThreadsInitialized) so existing
  call sites compile against modern CPython. Polyfills route to the
  PyConfig API where there's a meaningful replacement, return 0/no-op
  otherwise.

- New PEP 684 SubInterpreter wrapper (subinterp.go) — recommended
  primitive for tenant/per-worker isolation on 3.12+. Defaults follow
  PEP 684 (own GIL, threads on, no daemon threads, refuse non-isolated
  C extensions). Falls back to Py_NewInterpreter on 3.11 and older.

- Free-threading awareness (PEP 703) — IsGILDisabled() reports at
  runtime whether the linked CPython was built with --disable-gil.

- pkg-config switched from "python3" to "python3-embed" so libpython
  links cleanly on Linux distros that split the embed flags.

- Module path: github.com/go-python/cpy3 -> github.com/hanzoai/cpy3.
  Go 1.21+. Package name (python3) unchanged.

- CI matrix: Python {3.11, 3.12, 3.13} x Go {1.21, 1.22} on Ubuntu.

License remains MIT. Original go-python team credited in HANZO_CHANGES.md
and every modified file's header.

* feat: JSON hot-path helper + 3.13t build script

- LoadSource + CallJSONFunctionByName: single-cgo-crossing helpers
  for the common "JSON-in, JSON-out" extension embedding contract.
  Roughly 2x faster than the long form via individual wrappers and
  stack-safer under sustained load (macOS SIGBUS at ~400k iters of
  the 9-call long form is mitigated by collapsing to 1 cgo call).
- ErrUnknownFunction: sentinel error for missing/non-callable
  lookups so callers can errors.Is.
- scripts/build-python313t.sh: idempotent local build of CPython
  3.13.x with --disable-gil. Writes a python3-embed.pc so callers
  steer via PKG_CONFIG_PATH.
- HANZO_CHANGES.md + README updated.
- .gitignore: exclude python313t-build/ python313t-src/.

* ci: split test runs by Python global state

TestSubInterpreter_* and TestCallJSONFunctionByName_* / TestLoadSource_*
share global Py state with the data-type/error tests. Running them
in the same `go test` invocation can hang on darwin after the
data-type tests have already touched the main interpreter. Split
into two test steps so each starts from a fresh process.

* test: move json_hotpath tests in-package + share Py init

Move from package python3_test to package python3 so they share
Py_Initialize() state with subinterp_test.go. Each test pins its
own OS thread, calls Py_Initialize (idempotent), creates a fresh
sub-interp, and unpins on return. No PyEval_SaveThread in teardown
— the sub-interp's Close already released the calling thread's GIL.

This eliminates a SIGSEGV-on-second-test pattern caused by trying
to PyEval_SaveThread a state that's already been saved.

* docs: README example for the JSON hot path

Show LoadSource + CallJSONFunctionByName end-to-end with luxfi/log
for context. The helpers themselves take no logger dependency.
2026-05-18 23:38:11 -07:00

289 lines
7.5 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/*
Unless explicitly stated otherwise all files in this repository are licensed
under the MIT License.
This product includes software developed at Datadog (https://www.datadoghq.com/).
Copyright 2018 Datadog, Inc.
*/
package python3
/*
#include "Python.h"
#include "macro.h"
*/
import "C"
import (
"fmt"
"unsafe"
)
// The argument for Py_SetProgramName, Py_SetPath and Py_SetPythonHome should point to a zero-terminated wide character string in static storage
// whose contents will not change for the duration of the program’s execution
var (
programName *C.wchar_t
pythonPath *C.wchar_t
pythonHome *C.wchar_t
)
//Py_Initialize : https://docs.python.org/3/c-api/init.html#c.Py_Initialize
func Py_Initialize() {
C.Py_Initialize()
}
//Py_InitializeEx : https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx
func Py_InitializeEx(initsigs bool) {
if initsigs {
C.Py_InitializeEx(1)
} else {
C.Py_InitializeEx(0)
}
}
//Py_IsInitialized : https://docs.python.org/3/c-api/init.html#c.Py_IsInitialized
func Py_IsInitialized() bool {
return C.Py_IsInitialized() != 0
}
//Py_FinalizeEx : https://docs.python.org/3/c-api/init.html#c.Py_FinalizeEx
func Py_FinalizeEx() int {
return int(C.Py_FinalizeEx())
}
//Py_Finalize : https://docs.python.org/3/c-api/init.html#c.Py_Finalize
func Py_Finalize() {
C.Py_Finalize()
}
//Py_SetStandardStreamEncoding : https://docs.python.org/3/c-api/init.html#c.Py_SetStandardStreamEncoding
func Py_SetStandardStreamEncoding(encoding, errors string) int {
cencoding := C.CString(encoding)
defer C.free(unsafe.Pointer(cencoding))
cerrors := C.CString(errors)
defer C.free(unsafe.Pointer(cerrors))
return int(C.Py_SetStandardStreamEncoding(cencoding, cerrors))
}
//Py_SetProgramName : https://docs.python.org/3/c-api/init.html#c.Py_SetProgramName
func Py_SetProgramName(name string) error {
cname := C.CString(name)
defer C.free(unsafe.Pointer(cname))
newProgramName := C.Py_DecodeLocale(cname, nil)
if newProgramName == nil {
return fmt.Errorf("fail to call Py_DecodeLocale on '%s'", name)
}
C.Py_SetProgramName(newProgramName)
//no operation is performed if nil
C.PyMem_RawFree(unsafe.Pointer(programName))
programName = newProgramName
return nil
}
//Py_GetProgramName : https://docs.python.org/3/c-api/init.html#c.Py_GetProgramName
func Py_GetProgramName() (string, error) {
wcname := C.Py_GetProgramName()
if wcname == nil {
return "", nil
}
cname := C.Py_EncodeLocale(wcname, nil)
if cname == nil {
return "", fmt.Errorf("fail to call Py_EncodeLocale")
}
defer C.PyMem_Free(unsafe.Pointer(cname))
return C.GoString(cname), nil
}
//Py_GetPrefix : https://docs.python.org/3/c-api/init.html#c.Py_GetPrefix
func Py_GetPrefix() (string, error) {
wcname := C.Py_GetPrefix()
if wcname == nil {
return "", nil
}
cname := C.Py_EncodeLocale(wcname, nil)
if cname == nil {
return "", fmt.Errorf("fail to call Py_EncodeLocale")
}
defer C.PyMem_Free(unsafe.Pointer(cname))
return C.GoString(cname), nil
}
//Py_GetExecPrefix : https://docs.python.org/3/c-api/init.html#c.Py_GetExecPrefix
func Py_GetExecPrefix() (string, error) {
wcname := C.Py_GetExecPrefix()
if wcname == nil {
return "", nil
}
cname := C.Py_EncodeLocale(wcname, nil)
if cname == nil {
return "", fmt.Errorf("fail to call Py_EncodeLocale")
}
defer C.PyMem_Free(unsafe.Pointer(cname))
return C.GoString(cname), nil
}
//Py_GetProgramFullPath : https://docs.python.org/3/c-api/init.html#c.Py_GetProgramFullPath
func Py_GetProgramFullPath() (string, error) {
wcname := C.Py_GetProgramFullPath()
if wcname == nil {
return "", nil
}
cname := C.Py_EncodeLocale(wcname, nil)
if cname == nil {
return "", fmt.Errorf("fail to call Py_EncodeLocale")
}
defer C.PyMem_Free(unsafe.Pointer(cname))
return C.GoString(cname), nil
}
//Py_GetPath : https://docs.python.org/3/c-api/init.html#c.Py_GetPath
func Py_GetPath() (string, error) {
wcname := C.Py_GetPath()
if wcname == nil {
return "", nil
}
cname := C.Py_EncodeLocale(wcname, nil)
if cname == nil {
return "", fmt.Errorf("fail to call Py_EncodeLocale")
}
defer C.PyMem_Free(unsafe.Pointer(cname))
return C.GoString(cname), nil
}
//Py_SetPath : https://docs.python.org/3/c-api/init.html#c.Py_SetPath
func Py_SetPath(path string) error {
cpath := C.CString(path)
defer C.free(unsafe.Pointer(cpath))
newPath := C.Py_DecodeLocale(cpath, nil)
if newPath == nil {
return fmt.Errorf("fail to call Py_DecodeLocale on '%s'", path)
}
C.Py_SetPath(newPath)
C.PyMem_RawFree(unsafe.Pointer(pythonPath))
pythonHome = newPath
return nil
}
//Py_GetVersion : https://docs.python.org/3/c-api/init.html#c.Py_GetVersion
func Py_GetVersion() string {
cversion := C.Py_GetVersion()
return C.GoString(cversion)
}
//Py_GetPlatform : https://docs.python.org/3/c-api/init.html#c.Py_GetPlatform
func Py_GetPlatform() string {
cplatform := C.Py_GetPlatform()
return C.GoString(cplatform)
}
//Py_GetCopyright : https://docs.python.org/3/c-api/init.html#c.Py_GetCopyright
func Py_GetCopyright() string {
ccopyright := C.Py_GetCopyright()
return C.GoString(ccopyright)
}
//Py_GetCompiler : https://docs.python.org/3/c-api/init.html#c.Py_GetCompiler
func Py_GetCompiler() string {
ccompiler := C.Py_GetCompiler()
return C.GoString(ccompiler)
}
//Py_GetBuildInfo : https://docs.python.org/3/c-api/init.html#c.Py_GetBuildInfo
func Py_GetBuildInfo() string {
cbuildInfo := C.Py_GetBuildInfo()
return C.GoString(cbuildInfo)
}
//PySys_SetArgvEx : https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx
func PySys_SetArgvEx(args []string, updatepath bool) error {
argc := C.int(len(args))
argv := make([]*C.wchar_t, argc, argc)
for i, arg := range args {
carg := C.CString(arg)
defer C.free(unsafe.Pointer(carg))
warg := C.Py_DecodeLocale(carg, nil)
if warg == nil {
return fmt.Errorf("fail to call Py_DecodeLocale on '%s'", arg)
}
// Py_DecodeLocale requires a call to PyMem_RawFree to free the memory
defer C.PyMem_RawFree(unsafe.Pointer(warg))
argv[i] = warg
}
if updatepath {
C.PySys_SetArgvEx(argc, (**C.wchar_t)(unsafe.Pointer(&argv[0])), 1)
} else {
C.PySys_SetArgvEx(argc, (**C.wchar_t)(unsafe.Pointer(&argv[0])), 0)
}
return nil
}
//PySys_SetArgv : https://docs.python.org/3/c-api/init.html#c.PySys_SetArgv
func PySys_SetArgv(args []string) error {
argc := C.int(len(args))
argv := make([]*C.wchar_t, argc, argc)
for i, arg := range args {
carg := C.CString(arg)
defer C.free(unsafe.Pointer(carg))
warg := C.Py_DecodeLocale(carg, nil)
if warg == nil {
return fmt.Errorf("fail to call Py_DecodeLocale on '%s'", arg)
}
// Py_DecodeLocale requires a call to PyMem_RawFree to free the memory
defer C.PyMem_RawFree(unsafe.Pointer(warg))
argv[i] = warg
}
C.PySys_SetArgv(argc, (**C.wchar_t)(unsafe.Pointer(&argv[0])))
return nil
}
//Py_SetPythonHome : https://docs.python.org/3/c-api/init.html#c.Py_SetPythonHome
func Py_SetPythonHome(home string) error {
chome := C.CString(home)
defer C.free(unsafe.Pointer(chome))
newHome := C.Py_DecodeLocale(chome, nil)
if newHome == nil {
return fmt.Errorf("fail to call Py_DecodeLocale on '%s'", home)
}
C.Py_SetPythonHome(newHome)
C.PyMem_RawFree(unsafe.Pointer(pythonHome))
pythonHome = newHome
return nil
}
//Py_GetPythonHome : https://docs.python.org/3/c-api/init.html#c.Py_GetPythonHome
func Py_GetPythonHome() (string, error) {
wchome := C.Py_GetPythonHome()
if wchome == nil {
return "", nil
}
chome := C.Py_EncodeLocale(wchome, nil)
if chome == nil {
return "", fmt.Errorf("fail to call Py_EncodeLocale")
}
defer C.PyMem_Free(unsafe.Pointer(chome))
return C.GoString(chome), nil
}