Merge pull request #871 from crazywhalecc/fix/frankenphp-dynamic-exports

build frankenphp and embed after shared extensions

.editorconfig

@@ -12,5 +12,5 @@ trim_trailing_whitespace = true
 trim_trailing_whitespace = false
 indent_size = 2
 
-[*.{yml,yaml, vue}]
+[*.{yml,yaml,vue,ts}]
 indent_size = 2

.github/workflows/vitepress-deploy.yml

@@ -21,7 +21,6 @@ jobs:
 
       - uses: actions/setup-node@v3
         with:
-          node-version: 18
           cache: yarn
 
       - run: yarn install --frozen-lockfile

config/lib.json

@@ -1,9 +1,6 @@
 {
     "lib-base": {
-        "type": "root",
-        "lib-depends-unix": [
-            "pkg-config"
-        ]
+        "type": "root"
     },
     "php": {
         "type": "root",

config/pkg.json

@@ -1,4 +1,16 @@
 {
+    "go-xcaddy-aarch64-linux": {
+        "type": "custom"
+    },
+    "go-xcaddy-aarch64-macos": {
+        "type": "custom"
+    },
+    "go-xcaddy-x86_64-linux": {
+        "type": "custom"
+    },
+    "go-xcaddy-x86_64-macos": {
+        "type": "custom"
+    },
     "musl-toolchain-aarch64-linux": {
         "type": "url",
         "url": "https://dl.static-php.dev/static-php-cli/deps/musl-toolchain/aarch64-musl-toolchain.tgz"
@@ -15,6 +27,38 @@
             "nasm-2.16.01/ndisasm.exe": "{php_sdk_path}/bin/ndisasm.exe"
         }
     },
+    "pkg-config-aarch64-linux": {
+        "type": "ghrel",
+        "repo": "static-php/static-php-cli-hosted",
+        "match": "pkg-config-aarch64-linux-musl-1.2.5.txz",
+        "extract-files": {
+            "bin/pkg-config": "{pkg_root_path}/bin/pkg-config"
+        }
+    },
+    "pkg-config-aarch64-macos": {
+        "type": "ghrel",
+        "repo": "static-php/static-php-cli-hosted",
+        "match": "pkg-config-aarch64-darwin.txz",
+        "extract-files": {
+            "bin/pkg-config": "{pkg_root_path}/bin/pkg-config"
+        }
+    },
+    "pkg-config-x86_64-linux": {
+        "type": "ghrel",
+        "repo": "static-php/static-php-cli-hosted",
+        "match": "pkg-config-x86_64-linux-musl-1.2.5.txz",
+        "extract-files": {
+            "bin/pkg-config": "{pkg_root_path}/bin/pkg-config"
+        }
+    },
+    "pkg-config-x86_64-macos": {
+        "type": "ghrel",
+        "repo": "static-php/static-php-cli-hosted",
+        "match": "pkg-config-x86_64-darwin.txz",
+        "extract-files": {
+            "bin/pkg-config": "{pkg_root_path}/bin/pkg-config"
+        }
+    },
     "strawberry-perl-x86_64-win": {
         "type": "url",
         "url": "https://github.com/StrawberryPerl/Perl-Dist-Strawberry/releases/download/SP_5380_5361/strawberry-perl-5.38.0.1-64bit-portable.zip"
@@ -43,30 +87,18 @@
             "upx-*-win64/upx.exe": "{pkg_root_path}/bin/upx.exe"
         }
     },
-    "go-xcaddy-x86_64-linux": {
+    "zig-aarch64-linux": {
         "type": "custom"
     },
-    "go-xcaddy-aarch64-linux": {
-        "type": "custom"
-    },
-    "go-xcaddy-x86_64-macos": {
-        "type": "custom"
-    },
-    "go-xcaddy-aarch64-macos": {
+    "zig-aarch64-macos": {
         "type": "custom"
     },
     "zig-x86_64-linux": {
         "type": "custom"
     },
-    "zig-aarch64-linux": {
-        "type": "custom"
-    },
     "zig-x86_64-macos": {
         "type": "custom"
     },
-    "zig-aarch64-macos": {
-        "type": "custom"
-    },
     "zig-x86_64-win": {
         "type": "custom"
     }

docs/.vitepress/config.ts

@@ -51,6 +51,14 @@ export default {
     footer: {
       message: 'Released under the MIT License.',
       copyright: 'Copyright © 2023-present crazywhalecc'
-    }
+    },
+    search: {
+      provider: 'algolia',
+      options: {
+        appId: 'IHJHUB1SF1',
+        apiKey: '8266d31cc2ffbd0e059f1c6e5bdaf8fc',
+        indexName: 'static-php docs',
+      },
+    },
   }
 }

src/SPC/builder/freebsd/BSDBuilder.php

@@ -25,10 +25,6 @@ class BSDBuilder extends UnixBuilderBase
         f_putenv('CXX=' . $this->getOption('cxx', 'clang++'));
         // set PATH
         f_putenv('PATH=' . BUILD_ROOT_PATH . '/bin:' . getenv('PATH'));
-        // set PKG_CONFIG
-        f_putenv('PKG_CONFIG=' . BUILD_ROOT_PATH . '/bin/pkg-config');
-        // set PKG_CONFIG_PATH
-        f_putenv('PKG_CONFIG_PATH=' . BUILD_LIB_PATH . '/pkgconfig/');
 
         // set arch (default: current)
         $this->setOptionIfNotExist('arch', php_uname('m'));
@@ -122,6 +118,11 @@ class BSDBuilder extends UnixBuilderBase
             }
             $this->buildEmbed();
         }
+        $shared_extensions = array_map('trim', array_filter(explode(',', $this->getOption('build-shared'))));
+        if (!empty($shared_extensions)) {
+            logger()->info('Building shared extensions ...');
+            $this->buildSharedExts();
+        }
         if ($enableFrankenphp) {
             logger()->info('building frankenphp');
             $this->buildFrankenphp();

src/SPC/builder/linux/LinuxBuilder.php

@@ -142,6 +142,12 @@ class LinuxBuilder extends UnixBuilderBase
             }
             $this->buildEmbed();
         }
+        // build dynamic extensions if needed, must happen before building FrankenPHP to make sure we export all necessary, undefined symbols
+        $shared_extensions = array_map('trim', array_filter(explode(',', $this->getOption('build-shared'))));
+        if (!empty($shared_extensions)) {
+            logger()->info('Building shared extensions ...');
+            $this->buildSharedExts();
+        }
         if ($enableFrankenphp) {
             logger()->info('building frankenphp');
             $this->buildFrankenphp();
@@ -312,6 +318,8 @@ class LinuxBuilder extends UnixBuilderBase
         if (getenv('SPC_CMD_VAR_PHP_EMBED_TYPE') === 'static') {
             $AR = getenv('AR') ?: 'ar';
             f_passthru("{$AR} -t " . BUILD_LIB_PATH . "/libphp.a | grep '\\.a$' | xargs -n1 {$AR} d " . BUILD_LIB_PATH . '/libphp.a');
+            // export dynamic symbols
+            SystemUtil::exportDynamicSymbols(BUILD_LIB_PATH . '/libphp.a');
         }
 
         if (!$this->getOption('no-strip', false) && file_exists(BUILD_LIB_PATH . '/' . $realLibName)) {

src/SPC/builder/linux/library/liburing.php

@@ -51,8 +51,7 @@ class liburing extends LinuxLibraryBase
                 $use_libc ? '--use-libc' : '',
             )
             ->configure()
-            ->make('library', with_clean: false)
-            ->exec("rm -rf {$this->getLibDir()}/liburing*.so*");
+            ->make('library', 'install ENABLE_SHARED=0', with_clean: false);
 
         $this->patchPkgconfPrefix();
     }

src/SPC/builder/macos/MacOSBuilder.php

@@ -156,6 +156,11 @@ class MacOSBuilder extends UnixBuilderBase
             }
             $this->buildEmbed();
         }
+        $shared_extensions = array_map('trim', array_filter(explode(',', $this->getOption('build-shared'))));
+        if (!empty($shared_extensions)) {
+            logger()->info('Building shared extensions ...');
+            $this->buildSharedExts();
+        }
         if ($enableFrankenphp) {
             logger()->info('building frankenphp');
             $this->buildFrankenphp();
@@ -247,6 +252,8 @@ class MacOSBuilder extends UnixBuilderBase
         if (getenv('SPC_CMD_VAR_PHP_EMBED_TYPE') === 'static') {
             $AR = getenv('AR') ?: 'ar';
             f_passthru("{$AR} -t " . BUILD_LIB_PATH . "/libphp.a | grep '\\.a$' | xargs -n1 {$AR} d " . BUILD_LIB_PATH . '/libphp.a');
+            // export dynamic symbols
+            SystemUtil::exportDynamicSymbols(BUILD_LIB_PATH . '/libphp.a');
         }
         $this->patchPhpScripts();
     }

src/SPC/builder/traits/UnixSystemUtilTrait.php

@@ -4,15 +4,92 @@ declare(strict_types=1);
 
 namespace SPC\builder\traits;
 
-/**
- * Unix 系统的工具函数 Trait，适用于 Linux、macOS
- */
+use SPC\exception\ExecutionException;
+use SPC\exception\SPCInternalException;
+use SPC\exception\WrongUsageException;
+use SPC\toolchain\ToolchainManager;
+use SPC\toolchain\ZigToolchain;
+use SPC\util\SPCTarget;
+
 trait UnixSystemUtilTrait
 {
     /**
-     * @param  string      $name  命令名称
-     * @param  array       $paths 寻找的目标路径（如果不传入，则使用环境变量 PATH）
-     * @return null|string 找到了返回命令路径，找不到返回 null
+     * Export static library dynamic symbols to a .dynsym file.
+     * It will export to "/path/to/libxxx.a.dynsym".
+     *
+     * @param string $lib_file Static library file path (e.g. /path/to/libxxx.a)
+     */
+    public static function exportDynamicSymbols(string $lib_file): void
+    {
+        // check
+        if (!is_file($lib_file)) {
+            throw new WrongUsageException("The lib archive file {$lib_file} does not exist, please build it first.");
+        }
+        // shell out
+        $cmd = 'nm -g --defined-only -P ' . escapeshellarg($lib_file);
+        $result = shell()->execWithResult($cmd);
+        if ($result[0] !== 0) {
+            throw new ExecutionException($cmd, 'Failed to get defined symbols from ' . $lib_file);
+        }
+        // parse shell output and filter
+        $defined = [];
+        foreach ($result[1] as $line) {
+            $line = trim($line);
+            if ($line === '' || str_ends_with($line, '.o:') || str_ends_with($line, '.o]:')) {
+                continue;
+            }
+            $name = strtok($line, " \t");
+            if (!$name) {
+                continue;
+            }
+            $name = preg_replace('/@.*$/', '', $name);
+            if ($name !== '' && $name !== false) {
+                $defined[] = $name;
+            }
+        }
+        $defined = array_unique($defined);
+        sort($defined);
+        // export
+        if (SPCTarget::getTargetOS() === 'Linux') {
+            file_put_contents("{$lib_file}.dynsym", "{\n" . implode("\n", array_map(fn ($x) => "  {$x};", $defined)) . "};\n");
+        } else {
+            file_put_contents("{$lib_file}.dynsym", implode("\n", $defined) . "\n");
+        }
+    }
+
+    /**
+     * Get linker flag to export dynamic symbols from a static library.
+     *
+     * @param  string      $lib_file Static library file path (e.g. /path/to/libxxx.a)
+     * @return null|string Linker flag to export dynamic symbols, null if no .dynsym file found
+     */
+    public static function getDynamicExportedSymbols(string $lib_file): ?string
+    {
+        $symbol_file = "{$lib_file}.dynsym";
+        if (!is_file($symbol_file)) {
+            self::exportDynamicSymbols($lib_file);
+        }
+        if (!is_file($symbol_file)) {
+            throw new SPCInternalException("The symbol file {$symbol_file} does not exist, please check if nm command is available.");
+        }
+        // https://github.com/ziglang/zig/issues/24662
+        if (ToolchainManager::getToolchainClass() === ZigToolchain::class) {
+            return '-Wl,--export-dynamic';
+        }
+        // macOS
+        if (SPCTarget::getTargetOS() !== 'Linux') {
+            return "-Wl,-exported_symbols_list,{$symbol_file}";
+        }
+        return "-Wl,--dynamic-list={$symbol_file}";
+    }
+
+    /**
+     * Find a command in given paths or system PATH.
+     * If $name is an absolute path, check if it exists.
+     *
+     * @param  string      $name  Command name or absolute path
+     * @param  array       $paths Paths to search, if empty, use system PATH
+     * @return null|string Absolute path of the command if found, null otherwise
      */
     public static function findCommand(string $name, array $paths = []): ?string
     {
@@ -31,6 +108,8 @@ trait UnixSystemUtilTrait
     }
 
     /**
+     * Make environment variable string for shell command.
+     *
      * @param  array  $vars Variables, like: ["CFLAGS" => "-Ixxx"]
      * @return string like: CFLAGS="-Ixxx"
      */

src/SPC/builder/unix/UnixBuilderBase.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace SPC\builder\unix;
 
 use SPC\builder\BuilderBase;
+use SPC\builder\linux\SystemUtil as LinuxSystemUtil;
 use SPC\exception\SPCInternalException;
 use SPC\exception\ValidationException;
 use SPC\exception\WrongUsageException;
@@ -137,6 +138,7 @@ abstract class UnixBuilderBase extends BuilderBase
             if (SPCTarget::isStatic()) {
                 $lens .= ' -static';
             }
+            $dynamic_exports = '';
             // if someone changed to EMBED_TYPE=shared, we need to add LD_LIBRARY_PATH
             if (getenv('SPC_CMD_VAR_PHP_EMBED_TYPE') === 'shared') {
                 if (PHP_OS_FAMILY === 'Darwin') {
@@ -151,8 +153,13 @@ abstract class UnixBuilderBase extends BuilderBase
                 foreach (glob(BUILD_LIB_PATH . "/libphp*.{$suffix}") as $file) {
                     unlink($file);
                 }
+                // calling linux system util in other unix OS is okay
+                if ($dynamic_exports = LinuxSystemUtil::getDynamicExportedSymbols(BUILD_LIB_PATH . '/libphp.a')) {
+                    $dynamic_exports = ' ' . $dynamic_exports;
+                }
             }
-            [$ret, $out] = shell()->cd($sample_file_path)->execWithResult(getenv('CC') . ' -o embed embed.c ' . $lens);
+            $cc = getenv('CC');
+            [$ret, $out] = shell()->cd($sample_file_path)->execWithResult("{$cc} -o embed embed.c {$lens} {$dynamic_exports}");
             if ($ret !== 0) {
                 throw new ValidationException(
                     'embed failed sanity check: build failed. Error message: ' . implode("\n", $out),
@@ -267,15 +274,20 @@ abstract class UnixBuilderBase extends BuilderBase
         ), true);
         $frankenPhpVersion = $releaseInfo['tag_name'];
         $libphpVersion = $this->getPHPVersion();
+        $dynamic_exports = '';
         if (getenv('SPC_CMD_VAR_PHP_EMBED_TYPE') === 'shared') {
-            $libphpVersion = preg_replace('/\.\d$/', '', $libphpVersion);
+            $libphpVersion = preg_replace('/\.\d+$/', '', $libphpVersion);
+        } else {
+            if ($dynamicSymbolsArgument = LinuxSystemUtil::getDynamicExportedSymbols(BUILD_LIB_PATH . '/libphp.a')) {
+                $dynamic_exports = ' ' . $dynamicSymbolsArgument;
+            }
         }
         $debugFlags = $this->getOption('no-strip') ? '-w -s ' : '';
-        $extLdFlags = "-extldflags '-pie'";
+        $extLdFlags = "-extldflags '-pie{$dynamic_exports}'";
         $muslTags = '';
         $staticFlags = '';
         if (SPCTarget::isStatic()) {
-            $extLdFlags = "-extldflags '-static-pie -Wl,-z,stack-size=0x80000'";
+            $extLdFlags = "-extldflags '-static-pie -Wl,-z,stack-size=0x80000{$dynamic_exports}'";
             $muslTags = 'static_build,';
             $staticFlags = '-static-pie';
         }

src/SPC/command/BuildPHPCommand.php

@@ -211,12 +211,6 @@ class BuildPHPCommand extends BuildCommand
         // start to build
         $builder->buildPHP($rule);
 
-        // build dynamic extensions if needed
-        if (!empty($shared_extensions)) {
-            logger()->info('Building shared extensions ...');
-            $builder->buildSharedExts();
-        }
-
         $builder->testPHP($rule);
 
         // compile stopwatch :P

src/SPC/doctor/item/PkgConfigCheck.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+namespace SPC\doctor\item;
+
+use SPC\doctor\AsCheckItem;
+use SPC\doctor\AsFixItem;
+use SPC\doctor\CheckResult;
+use SPC\doctor\OptionalCheck;
+use SPC\store\PackageManager;
+use SPC\util\PkgConfigUtil;
+
+#[OptionalCheck([self::class, 'optionalCheck'])]
+class PkgConfigCheck
+{
+    public static function optionalCheck(): bool
+    {
+        return PHP_OS_FAMILY !== 'Windows';
+    }
+
+    /** @noinspection PhpUnused */
+    #[AsCheckItem('if pkg-config is installed or built', level: 800)]
+    public function checkPkgConfig(): CheckResult
+    {
+        if (!($pkgconf = PkgConfigUtil::findPkgConfig())) {
+            return CheckResult::fail('pkg-config is not installed', 'install-pkgconfig');
+        }
+        return CheckResult::ok($pkgconf);
+    }
+
+    #[AsFixItem('install-pkgconfig')]
+    public function installPkgConfig(): bool
+    {
+        PackageManager::installPackage('pkg-config');
+        return true;
+    }
+}

src/SPC/toolchain/ToolchainManager.php

@@ -7,6 +7,7 @@ namespace SPC\toolchain;
 use SPC\builder\linux\SystemUtil;
 use SPC\exception\WrongUsageException;
 use SPC\util\GlobalEnvManager;
+use SPC\util\PkgConfigUtil;
 use SPC\util\SPCTarget;
 
 class ToolchainManager
@@ -56,6 +57,15 @@ class ToolchainManager
         if (SPCTarget::getLibc() === 'glibc' && SystemUtil::isMuslDist()) {
             throw new WrongUsageException('You are linking against glibc dynamically, which is only supported on glibc distros.');
         }
+
+        // init pkg-config for unix
+        if (is_unix()) {
+            if (($found = PkgConfigUtil::findPkgConfig()) === null) {
+                throw new WrongUsageException('Cannot find pkg-config executable. Please run `doctor` to fix this.');
+            }
+            GlobalEnvManager::putenv("PKG_CONFIG={$found}");
+        }
+
         $toolchain = getenv('SPC_TOOLCHAIN');
         /* @var ToolchainInterface $toolchain */
         $instance = new $toolchain();

src/SPC/util/GlobalEnvManager.php

@@ -39,8 +39,8 @@ class GlobalEnvManager
         // Define env vars for unix
         if (is_unix()) {
             self::addPathIfNotExists(BUILD_BIN_PATH);
-            self::putenv('PKG_CONFIG=' . BUILD_BIN_PATH . '/pkg-config');
-            self::putenv('PKG_CONFIG_PATH=' . BUILD_ROOT_PATH . '/lib/pkgconfig');
+            self::addPathIfNotExists(PKG_ROOT_PATH . '/bin');
+            self::putenv('PKG_CONFIG_PATH=' . BUILD_LIB_PATH . '/pkgconfig');
         }
 
         $ini = self::readIniFile();

src/SPC/util/PkgConfigUtil.php

@@ -14,6 +14,28 @@ use SPC\exception\ExecutionException;
  */
 class PkgConfigUtil
 {
+    /**
+     * Find the pkg-config executable which is compatible with static builds.
+     *
+     * @return null|string Path to pkg-config executable, or null if not found
+     */
+    public static function findPkgConfig(): ?string
+    {
+        // Find pkg-config executable
+        $find_list = [
+            PKG_ROOT_PATH . '/bin/pkg-config',
+            BUILD_BIN_PATH . '/pkg-config',
+        ];
+        $found = null;
+        foreach ($find_list as $file) {
+            if (file_exists($file) && is_executable($file)) {
+                $found = $file;
+                break;
+            }
+        }
+        return $found;
+    }
+
     /**
      * Returns the version of a module.
      * This method uses `pkg-config --modversion` to get the version of the specified module.

src/SPC/util/executor/UnixCMakeExecutor.php

@@ -8,6 +8,7 @@ use SPC\builder\freebsd\library\BSDLibraryBase;
 use SPC\builder\linux\library\LinuxLibraryBase;
 use SPC\builder\macos\library\MacOSLibraryBase;
 use SPC\store\FileSystem;
+use SPC\util\PkgConfigUtil;
 use SPC\util\shell\UnixShell;
 
 /**
@@ -184,6 +185,7 @@ class UnixCMakeExecutor extends Executor
         $cxx = getenv('CCX');
         logger()->debug("making cmake tool chain file for {$os} {$target_arch} with CFLAGS='{$cflags}'");
         $root = BUILD_ROOT_PATH;
+        $pkgConfigExecutable = PkgConfigUtil::findPkgConfig();
         $ccLine = '';
         if ($cc) {
             $ccLine = 'SET(CMAKE_C_COMPILER ' . $cc . ')';
@@ -202,7 +204,7 @@ SET(CMAKE_PREFIX_PATH "{$root}")
 SET(CMAKE_INSTALL_PREFIX "{$root}")
 SET(CMAKE_INSTALL_LIBDIR "lib")
 
-set(PKG_CONFIG_EXECUTABLE "{$root}/bin/pkg-config")
+set(PKG_CONFIG_EXECUTABLE "{$pkgConfigExecutable}")
 list(APPEND PKG_CONFIG_EXECUTABLE "--static")
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)

src/globals/test-extensions.php

@@ -13,9 +13,9 @@ declare(strict_types=1);
 
 // test php version (8.1 ~ 8.4 available, multiple for matrix)
 $test_php_version = [
-    '8.1',
-    '8.2',
-    '8.3',
+    // '8.1',
+    // '8.2',
+    // '8.3',
     '8.4',
     // '8.5',
     // 'git',
@@ -28,9 +28,9 @@ $test_os = [
     'macos-15', // bin/spc for arm64
     'ubuntu-latest', // bin/spc-alpine-docker for x86_64
     'ubuntu-22.04', // bin/spc-gnu-docker for x86_64
-    'ubuntu-24.04', // bin/spc for x86_64
+    // 'ubuntu-24.04', // bin/spc for x86_64
     'ubuntu-22.04-arm', // bin/spc-gnu-docker for arm64
-    'ubuntu-24.04-arm', // bin/spc for arm64
+    // 'ubuntu-24.04-arm', // bin/spc for arm64
     // 'windows-latest', // .\bin\spc.ps1
 ];
 
@@ -50,13 +50,13 @@ $prefer_pre_built = false;
 
 // If you want to test your added extensions and libs, add below (comma separated, example `bcmath,openssl`).
 $extensions = match (PHP_OS_FAMILY) {
-    'Linux', 'Darwin' => 'swoole,swoole-hook-mysql,swoole-hook-pgsql,swoole-hook-sqlite,swoole-hook-odbc,apcu,bcmath,bz2,calendar,ctype,curl,dba,dom,event,exif,fileinfo,filter,ftp,gd,gmp,iconv,imagick,intl,mbregex,mbstring,mysqli,mysqlnd,opcache,openssl,pcntl,pdo,pdo_mysql,pgsql,phar,posix,protobuf,readline,redis,session,shmop,simplexml,soap,sockets,sodium,sqlite3,swoole,sysvmsg,sysvsem,sysvshm,tokenizer,xml,xmlreader,xmlwriter,xsl,zip,zlib',
+    'Linux', 'Darwin' => 'bcmath',
     'Windows' => 'bcmath,bz2,calendar,ctype,curl,dom,exif,fileinfo,filter,ftp,iconv,xml,mbstring,mbregex,mysqlnd,openssl,pdo,pdo_mysql,pdo_sqlite,phar,session,simplexml,soap,sockets,sqlite3,tokenizer,xmlwriter,xmlreader,zlib,zip',
 };
 
 // If you want to test shared extensions, add them below (comma separated, example `bcmath,openssl`).
 $shared_extensions = match (PHP_OS_FAMILY) {
-    'Linux' => '',
+    'Linux' => 'zip',
     'Darwin' => '',
     'Windows' => '',
 };