add pgo capabilities v3 style

This commit is contained in:
henderkes
2026-05-11 19:06:40 +07:00
parent 743934d1fe
commit 7e6e9d869e
13 changed files with 600 additions and 107 deletions

View File

@@ -9,7 +9,10 @@ use StaticPHP\Doctor\Doctor;
use StaticPHP\Exception\ValidationException;
use StaticPHP\Package\PackageBuilder;
use StaticPHP\Package\PackageInstaller;
use StaticPHP\Registry\PackageLoader;
use StaticPHP\Util\DependencyResolver;
use StaticPHP\Util\FileSystem;
use StaticPHP\Util\Pgo\PgoContext;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Yaml\Exception\ParseException;
@@ -21,6 +24,8 @@ class CraftCommand extends BaseCommand
public function configure(): void
{
$this->addArgument('craft', null, 'Path to craft.yml file', WORKING_DIR . '/craft.yml');
$this->addOption('libs-only', null, null, 'Build only the libraries needed by the configured extensions (skip PHP and SAPI build).');
PgoContext::registerOptions($this);
}
public function handle(): int
@@ -83,23 +88,67 @@ class CraftCommand extends BaseCommand
FileSystem::resetDir(SOURCE_PATH);
}
$pgo = $this->getOption('libs-only') ? null : PgoContext::tryFromInput($this->input, $craft['sapi'], $build_options);
$starttime = microtime(true);
// run installer
$installer = new PackageInstaller($build_options);
ApplicationContext::get(PackageBuilder::class)->setArgument('extensions', implode(',', $craft['extensions']));
$installer->addBuildPackage('php');
if ($this->getOption('libs-only')) {
$with_suggests = (bool) ($craft['build-options']['with-suggests'] ?? false);
$libs = $this->resolveLibsForExtensions($craft, $with_suggests);
if ($libs === []) {
$this->output->writeln('<comment>No libraries needed for the configured extensions; nothing to do.</comment>');
return static::SUCCESS;
}
foreach ($libs as $lib) {
$installer->addBuildPackage($lib);
}
} else {
$installer->addBuildPackage('php');
}
$installer->run(true);
$usedtime = round(microtime(true) - $starttime, 1);
$tag = $pgo !== null ? " (PGO {$pgo->mode})" : '';
$this->output->writeln("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
$this->output->writeln("<info>✔ BUILD SUCCESSFUL ({$usedtime} s)</info>");
$this->output->writeln("<info>✔ BUILD SUCCESSFUL{$tag} ({$usedtime} s)</info>");
$this->output->writeln("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n");
if ($pgo !== null && $pgo->isInstrument()) {
$this->output->writeln("<comment>Next: exercise the instrumented binary, then re-run craft with --pgo to consume {$pgo->profileRoot}.</comment>");
}
$installer->printBuildPackageOutputs();
return static::SUCCESS;
}
/** @return list<string> library package names transitively required by the configured extensions */
private function resolveLibsForExtensions(array $craft, bool $include_suggests): array
{
$exts = array_merge($craft['extensions'], $craft['shared-extensions'] ?? []);
$ext_pkgs = array_map(fn ($x) => "ext-{$x}", $exts);
$extra = $craft['packages'] ?? [];
$resolved = DependencyResolver::resolve(
array_merge($ext_pkgs, $extra),
include_suggests: $include_suggests,
);
$libs = [];
foreach ($resolved as $pkg_name) {
if (str_starts_with($pkg_name, 'ext-') || !PackageLoader::hasPackage($pkg_name)) {
continue;
}
if (PackageLoader::getPackage($pkg_name)->getType() === 'library') {
$libs[] = $pkg_name;
}
}
return $libs;
}
/**
* Validate and parse craft.yml file to array.
*

View File

@@ -215,7 +215,7 @@ class PackageInstaller
if (!$is_to_build && $should_use_binary) {
// install binary
if ($this->interactive) {
InteractiveTerm::indicateProgress('Installing package: ' . ConsoleColor::yellow($package->getName()));
InteractiveTerm::indicateProgress('Installing ' . $this->kindLabel($package) . ': ' . ConsoleColor::yellow($package->getName()));
}
try {
// Start tracking for binary installation
@@ -227,17 +227,17 @@ class PackageInstaller
// Stop tracking on error
$this->tracker?->stopTracking();
if ($this->interactive) {
InteractiveTerm::finish('Installing binary package failed: ' . ConsoleColor::red($package->getName()), false);
InteractiveTerm::finish('Installing ' . $this->kindLabel($package) . ' failed: ' . ConsoleColor::red($package->getName()), false);
echo PHP_EOL;
}
throw $e;
}
if ($this->interactive) {
InteractiveTerm::finish('Installed binary package: ' . ConsoleColor::green($package->getName()) . ($status === SPC_STATUS_ALREADY_INSTALLED ? ' (already installed, skipped)' : ''));
InteractiveTerm::finish('Installed ' . $this->kindLabel($package) . ': ' . ConsoleColor::green($package->getName()) . ($status === SPC_STATUS_ALREADY_INSTALLED ? ' (already installed, skipped)' : ''));
}
} elseif ($is_to_build && $has_build_stage || $has_source && $has_build_stage) {
if ($this->interactive) {
InteractiveTerm::indicateProgress('Building package: ' . ConsoleColor::yellow($package->getName()));
InteractiveTerm::indicateProgress('Building ' . $this->kindLabel($package) . ': ' . ConsoleColor::yellow($package->getName()));
}
try {
// Start tracking for build
@@ -260,13 +260,13 @@ class PackageInstaller
// Stop tracking on error
$this->tracker?->stopTracking();
if ($this->interactive) {
InteractiveTerm::finish('Building package failed: ' . ConsoleColor::red($package->getName()), false);
InteractiveTerm::finish('Building ' . $this->kindLabel($package) . ' failed: ' . ConsoleColor::red($package->getName()), false);
echo PHP_EOL;
}
throw $e;
}
if ($this->interactive) {
InteractiveTerm::finish('Built package: ' . ConsoleColor::green($package->getName()) . ($status === SPC_STATUS_ALREADY_BUILT ? ' (already built, skipped)' : ''));
InteractiveTerm::finish('Built ' . $this->kindLabel($package) . ': ' . ConsoleColor::green($package->getName()) . ($status === SPC_STATUS_ALREADY_BUILT ? ' (already built, skipped)' : ''));
}
}
}
@@ -568,6 +568,16 @@ class PackageInstaller
return null;
}
private function kindLabel(Package $package): string
{
return match (true) {
$package instanceof PhpExtensionPackage => 'extension',
$package instanceof TargetPackage => 'target',
$package instanceof LibraryPackage => 'library',
default => 'package',
};
}
/**
* @param Package[] $packages
*/

View File

@@ -12,6 +12,7 @@ use StaticPHP\Exception\WrongUsageException;
use StaticPHP\Runtime\SystemTarget;
use StaticPHP\Toolchain\ToolchainManager;
use StaticPHP\Toolchain\ZigToolchain;
use StaticPHP\Util\FileSystem;
use StaticPHP\Util\GlobalEnvManager;
use StaticPHP\Util\SPCConfigUtil;
@@ -326,16 +327,29 @@ class PhpExtensionPackage extends Package
{
// phpize Makefile's _SHARED_LIBADD line misses our static archives — splice them in
$package->patchSharedLibAdd();
$extra_ldflags = (string) getenv('SPC_CMD_VAR_PHP_MAKE_EXTRA_LDFLAGS');
$makeArgs = $extra_ldflags !== '' ? 'EXTRA_LDFLAGS=' . escapeshellarg($extra_ldflags) : '';
shell()->cd($package->getSourceDir())
->setEnv($env)
->exec('make clean')
->exec("make -j{$builder->concurrency}")
->exec('make install');
->exec("make -j{$builder->concurrency} {$makeArgs}")
->exec("make install {$makeArgs}");
// install-modules deref'd libtool's `$ext.so → $ext-X.so` symlink into two regular files; restore the symlink.
if (preg_match('/-release\s+(\S+)/', $extra_ldflags, $m)) {
$name = $package->getExtensionName();
$unversioned = BUILD_MODULES_PATH . "/{$name}.so";
$versioned = BUILD_MODULES_PATH . "/{$name}-{$m[1]}.so";
if (file_exists($versioned) && file_exists($unversioned) && !is_link($unversioned)) {
unlink($unversioned);
symlink(basename($versioned), $unversioned);
}
}
}
public function patchSharedLibAdd(): void
{
$config = (new SPCConfigUtil())->getExtensionConfig($this);
$config = new SPCConfigUtil()->getExtensionConfig($this);
[$staticLibs, $sharedLibs] = $this->splitLibsIntoStaticAndShared($config['libs']);
$lstdcpp = str_contains($sharedLibs, '-l:libstdc++.a')
? '-l:libstdc++.a'
@@ -353,7 +367,7 @@ class PhpExtensionPackage extends Package
$current = trim($m[2]);
$merged = clean_spaces("{$current} {$staticLibs} {$lstdcpp}");
$merged = deduplicate_flags($merged);
\StaticPHP\Util\FileSystem::replaceFileRegex(
FileSystem::replaceFileRegex(
$makefile,
'/^(.*_SHARED_LIBADD\s*=.*)$/m',
$prefix . $merged
@@ -389,8 +403,10 @@ class PhpExtensionPackage extends Package
$this->runStage([$this, 'configureForUnix'], ['env' => $env]);
$this->runStage([$this, 'makeForUnix'], ['env' => $env]);
// process *.so file
$soFile = BUILD_MODULES_PATH . '/' . $this->getExtensionName() . '.so';
// libtool's -release X gives $name-X.so as the real file
$soFile = BUILD_MODULES_PATH . '/' . $this->getExtensionName()
. (preg_match('/-release\s+(\S+)/', (string) getenv('SPC_CMD_VAR_PHP_MAKE_EXTRA_LDFLAGS'), $m) ? "-{$m[1]}" : '')
. '.so';
if (!file_exists($soFile)) {
throw new ValidationException("Extension {$this->getExtensionName()} build failed: {$soFile} not found", validation_module: "Extension {$this->getExtensionName()} build");
}

View File

@@ -89,14 +89,17 @@ class Registry
self::$current_registry_name = $registry_name;
try {
// Load composer autoload if specified (for external registries with their own dependencies)
// resolve autoload manually — path-repo installs have no vendor/, FileSystem::fullpath would throw
if (isset($data['autoload']) && is_string($data['autoload'])) {
$autoload_path = FileSystem::fullpath($data['autoload'], dirname($registry_file));
$base = dirname($registry_file);
$autoload_path = FileSystem::isRelativePath($data['autoload'])
? rtrim($base, '/') . DIRECTORY_SEPARATOR . $data['autoload']
: $data['autoload'];
if (file_exists($autoload_path)) {
logger()->debug("Loading external autoload from: {$autoload_path}");
require_once $autoload_path;
} else {
logger()->warning("Autoload file not found: {$autoload_path}");
logger()->warning("Registry autoload not present, relying on consumer autoloader: {$autoload_path}");
}
}

View File

@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace StaticPHP\Util;
use StaticPHP\DI\ApplicationContext;
use Symfony\Component\Console\Helper\Helper;
use Symfony\Component\Console\Helper\ProgressIndicator;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\ConsoleOutput;
@@ -27,14 +28,19 @@ class InteractiveTerm
}
}
public static function success(string $message, bool $indent = false): void
public static function success(string $message, bool $indent = false, ?float $start_time = null): void
{
$no_ansi = ApplicationContext::get(InputInterface::class)?->getOption('no-ansi') ?? false;
$output = ApplicationContext::get(OutputInterface::class) ?? new ConsoleOutput();
if ($start_time !== null) {
$message .= ' (' . Helper::formatTime(microtime(true) - $start_time) . ')';
}
if ($output->isVerbose()) {
logger()->info(strip_ansi_colors($message));
} else {
$output->writeln(($no_ansi ? 'strip_ansi_colors' : 'strval')(ConsoleColor::green(($indent ? ' ' : '') . '✔ ') . $message));
// wipe the current indicator line so our persistent ✔ line doesn't get appended to a spinner
$clear = (self::$indicator !== null && $output->isDecorated() && !$no_ansi) ? "\x0D\x1B[2K" : '';
$output->writeln($clear . ($no_ansi ? 'strip_ansi_colors' : 'strval')(ConsoleColor::green(($indent ? ' ' : '') . '✔ ') . $message));
logger()->debug(strip_ansi_colors($message));
}
}

View File

@@ -0,0 +1,267 @@
<?php
declare(strict_types=1);
namespace StaticPHP\Util\Pgo;
use StaticPHP\Command\BaseCommand;
use StaticPHP\DI\ApplicationContext;
use StaticPHP\Exception\WrongUsageException;
use StaticPHP\Util\FileSystem;
use Symfony\Component\Console\Input\InputInterface;
final class PgoContext
{
public const string MODE_INSTRUMENT = 'instrument';
public const string MODE_CS_INSTRUMENT = 'cs-instrument';
public const string MODE_USE = 'use';
/**
* @var array<string, string>
*/
public const array TRAINABLE = [
'cli' => 'build-cli',
'micro' => 'build-micro',
'cgi' => 'build-cgi',
'fpm' => 'build-fpm',
'embed' => 'build-embed',
'frankenphp' => 'build-frankenphp',
];
public const array SHUTDOWN_PATCHES = [
'php-src' => 'spc_pgo_flush_php_main.patch',
'frankenphp' => 'spc_pgo_flush_frankenphp.patch',
];
/** @var list<string> */
private array $trainableSapis = [];
public function __construct(
public readonly string $mode,
public readonly string $profileRoot,
) {
if (!in_array($mode, [self::MODE_INSTRUMENT, self::MODE_CS_INSTRUMENT, self::MODE_USE], true)) {
throw new WrongUsageException("PgoContext: unknown mode '{$mode}'");
}
}
public static function registerOptions(BaseCommand $cmd): void
{
$cmd->addOption('pgi', null, null, 'PGO instrument pass: build with -fprofile-generate so the resulting binary writes .profraw on shutdown.');
$cmd->addOption('cs-pgi', null, null, 'PGO context-sensitive instrument pass: -fprofile-use=<existing.profdata> + -fcs-profile-generate. Requires a prior --pgi/--pgo cycle.');
$cmd->addOption('pgo', null, null, 'PGO use pass: merge the collected .profraw into .profdata, then rebuild with -fprofile-use.');
}
/**
* @param array<string> $sapis
* @param array<string, mixed> $build_options
*/
public static function tryFromInput(InputInterface $input, array $sapis, array &$build_options): ?self
{
$modes = array_filter(['pgi', 'cs-pgi', 'pgo'], fn ($m) => (bool) $input->getOption($m));
if (count($modes) > 1) {
throw new WrongUsageException('--pgi, --cs-pgi, and --pgo are mutually exclusive');
}
$picked = array_values($modes)[0] ?? null;
if ($picked === null) {
return null;
}
$mode = match ($picked) {
'pgi' => self::MODE_INSTRUMENT,
'cs-pgi' => self::MODE_CS_INSTRUMENT,
'pgo' => self::MODE_USE,
};
$ctx = new self($mode, BUILD_ROOT_PATH . '/pgo-data');
$ctx->setTrainableSapis($sapis);
match ($mode) {
self::MODE_INSTRUMENT => $ctx->setupInstrument(),
self::MODE_CS_INSTRUMENT => $ctx->setupCsInstrument(),
self::MODE_USE => $ctx->mergeProfiles(),
};
if ($ctx->isInstrument() || $ctx->isCsInstrument()) {
$build_options['no-strip'] = true;
}
ApplicationContext::set(self::class, $ctx);
return $ctx;
}
public function isInstrument(): bool
{
return $this->mode === self::MODE_INSTRUMENT;
}
public function isCsInstrument(): bool
{
return $this->mode === self::MODE_CS_INSTRUMENT;
}
public function isUse(): bool
{
return $this->mode === self::MODE_USE;
}
/**
* @param list<string> $sapis
*/
public function setTrainableSapis(array $sapis): void
{
$resolved = [];
foreach ($sapis as $sapi) {
$r = $this->resolveSapi($sapi);
if (!in_array($r, $resolved, true)) {
$resolved[] = $r;
}
}
if ($resolved === []) {
throw new WrongUsageException(
'PGO: no trainable SAPI selected; supply one of ' . implode(', ', array_keys(self::TRAINABLE))
);
}
$this->trainableSapis = $resolved;
}
/** @return list<string> */
public function trainableSapis(): array
{
return $this->trainableSapis;
}
/**
* Static-embed mode links libphp.a into frankenphp, sharing a single binary
* and profdata. Shared-embed keeps them separate.
*/
public function resolveSapi(string $sapi): string
{
if ($sapi === 'embed' && getenv('SPC_CMD_VAR_PHP_EMBED_TYPE') === 'static') {
return 'frankenphp';
}
return $sapi;
}
public function rawDir(string $sapi): string
{
return $this->profileRoot . '/' . $sapi;
}
public function csRawDir(string $sapi): string
{
return $this->profileRoot . '/cs-' . $sapi;
}
public function profDataFile(string $sapi): string
{
return $this->profileRoot . '/' . $sapi . '.profdata';
}
public function cflagsFor(string $sapi): string
{
$sapi = $this->resolveSapi($sapi);
if ($this->mode === self::MODE_USE && !is_file($this->profDataFile($sapi))) {
return '';
}
return match ($this->mode) {
self::MODE_INSTRUMENT => '-fprofile-generate=' . $this->rawDir($sapi)
. ' -fprofile-update=atomic',
self::MODE_CS_INSTRUMENT => '-fprofile-use=' . $this->profDataFile($sapi)
. ' -fcs-profile-generate=' . $this->csRawDir($sapi)
. ' -fprofile-update=atomic'
. ' -Wno-error=profile-instr-unprofiled'
. ' -Wno-error=profile-instr-out-of-date'
. ' -Wno-backend-plugin',
self::MODE_USE => '-fprofile-use=' . $this->profDataFile($sapi)
. ' -Wno-error=profile-instr-unprofiled'
. ' -Wno-error=profile-instr-out-of-date'
. ' -Wno-backend-plugin',
default => throw new WrongUsageException("PgoContext: unreachable mode '{$this->mode}'"),
};
}
public function ldflagsFor(string $sapi): string
{
$resolved = $this->resolveSapi($sapi);
$flags = $this->cflagsFor($sapi);
$patterns = ['/\s*-Wno-error=\S+/', '/\s*-Wno-backend-plugin/'];
if ($resolved === 'frankenphp') {
$patterns[] = '/\s*-fprofile-use=\S+/';
$patterns[] = '/\s*-fcs-profile-generate=\S+/';
}
return trim((string) preg_replace($patterns, '', $flags));
}
public function applyEnvFor(string $sapi): void
{
self::overwritePgoFlags('SPC_CMD_VAR_PHP_MAKE_EXTRA_CFLAGS', $this->cflagsFor($sapi));
self::overwritePgoFlags('SPC_CMD_VAR_PHP_MAKE_EXTRA_LDFLAGS_PROGRAM', $this->ldflagsFor($sapi));
}
public function setupInstrument(): void
{
FileSystem::removeDir($this->profileRoot);
FileSystem::createDir($this->profileRoot);
foreach ($this->trainableSapis as $sapi) {
FileSystem::createDir($this->rawDir($sapi));
}
}
public function setupCsInstrument(): void
{
foreach ($this->trainableSapis as $sapi) {
if (!is_file($this->profDataFile($sapi))) {
throw new WrongUsageException(
"PGO --phase=cs-instrument: missing {$sapi}.profdata; run --phase=instrument and --phase=use first"
);
}
FileSystem::createDir($this->csRawDir($sapi));
}
}
public function mergeProfiles(): void
{
if (trim((string) shell_exec('command -v llvm-profdata 2>/dev/null')) === '') {
throw new WrongUsageException('PGO --phase=use: llvm-profdata not on PATH');
}
foreach ($this->trainableSapis as $sapi) {
$this->mergeSapi($sapi);
}
}
private function mergeSapi(string $sapi): void
{
$raws = glob($this->rawDir($sapi) . '/*.profraw') ?: [];
$csRaws = glob($this->csRawDir($sapi) . '/*.profraw') ?: [];
if ($raws === [] && $csRaws === []) {
if ($sapi === 'frankenphp') {
logger()->warning(
'PGO --phase=use: no .profraw for frankenphp (cgo-glue PGO will be skipped); ' .
'run --phase=instrument, exercise frankenphp longer, then re-run --phase=use'
);
return;
}
throw new WrongUsageException(
"PGO --phase=use: no .profraw for {$sapi}; run --phase=instrument, exercise the binary, then re-run --phase=use"
);
}
$out = $this->profDataFile($sapi);
$inputs = array_merge($raws, $csRaws);
$argv = implode(' ', array_map('escapeshellarg', $inputs));
shell()->exec('llvm-profdata merge --failure-mode=warn -output=' . escapeshellarg($out) . ' ' . $argv);
if (!is_file($out) || filesize($out) === 0) {
throw new WrongUsageException("PGO --phase=use: empty merge output for {$sapi}");
}
logger()->info("PGO merged {$sapi}: " . filesize($out) . ' bytes');
}
private static function overwritePgoFlags(string $var, string $append): void
{
$cur = (string) getenv($var);
$cur = preg_replace('/\s*-f(cs-)?profile-(generate|use)=\S+/', '', $cur) ?? $cur;
$cur = preg_replace('/\s*-fprofile-update=atomic/', '', $cur) ?? $cur;
$cur = preg_replace('/\s*-Wno-error=profile-instr-\S+/', '', $cur) ?? $cur;
$cur = preg_replace('/\s*-Wno-backend-plugin/', '', $cur) ?? $cur;
f_putenv($var . '=' . trim(trim($cur) . ' ' . $append));
}
}

View File

@@ -67,7 +67,7 @@ class PkgConfigUtil
public static function getCflags(string $pkg_config_str): string
{
// get other things
$result = self::execWithResult("pkg-config --static --cflags-only-other {$pkg_config_str}");
$result = self::execWithResult("pkg-config --static --cflags {$pkg_config_str}");
return trim($result);
}