diff --git a/.github/workflows/build-unix.yml b/.github/workflows/build-unix.yml index 0ec63e24..457910da 100644 --- a/.github/workflows/build-unix.yml +++ b/.github/workflows/build-unix.yml @@ -6,10 +6,13 @@ on: os: required: true description: Build target OS + default: 'linux-x86_64' type: choice options: - 'linux-x86_64' - 'linux-aarch64' + - 'linux-x86_64-glibc' + - 'linux-aarch64-glibc' - 'macos-x86_64' - 'macos-aarch64' php-version: @@ -22,7 +25,6 @@ on: - '8.3' - '8.2' - '8.1' - - '8.0' extensions: description: Extensions to build (comma separated) required: true @@ -77,9 +79,19 @@ jobs: RUNS_ON="ubuntu-latest" ;; linux-aarch64) - DOWN_CMD="SPC_USE_ARCH=aarch64 ./bin/spc-alpine-docker download" - BUILD_CMD="SPC_USE_ARCH=aarch64 ./bin/spc-alpine-docker build" - RUNS_ON="ubuntu-latest" + DOWN_CMD="./bin/spc-alpine-docker download" + BUILD_CMD="./bin/spc-alpine-docker build" + RUNS_ON="ubuntu-24.04-arm" + ;; + linux-x86_64-glibc) + DOWN_CMD="./bin/spc-gnu-docker download" + BUILD_CMD="./bin/spc-gnu-docker build" + RUNS_ON="ubuntu-22.04" + ;; + linux-aarch64-glibc) + DOWN_CMD="./bin/spc-gnu-docker download" + BUILD_CMD="./bin/spc-gnu-docker build" + RUNS_ON="ubuntu-22.04-arm" ;; macos-x86_64) DOWN_CMD="composer update --no-dev --classmap-authoritative && ./bin/spc doctor --auto-fix && ./bin/spc download" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 29f195bb..7648873a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -176,18 +176,18 @@ jobs: run: composer update -q --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist - name: "Run Build Tests (doctor)" - run: bin/spc doctor --auto-fix --debug + run: php src/globals/test-extensions.php doctor_cmd ${{ matrix.os }} ${{ matrix.php }} - name: "Prepare UPX for Windows" - if: matrix.os == 'windows-latest' + if: ${{ startsWith(matrix.os, 'windows-') }} run: | - bin/spc install-pkg upx + php src/globals/test-extensions.php install_upx_cmd ${{ matrix.os }} ${{ matrix.php }} echo "UPX_CMD=$(php src/globals/test-extensions.php upx)" >> $env:GITHUB_ENV - name: "Prepare UPX for Linux" - if: matrix.os == 'ubunut-latest' + if: ${{ startsWith(matrix.os, 'ubuntu-') }} run: | - bin/spc install-pkg upx + php src/globals/test-extensions.php install_upx_cmd ${{ matrix.os }} ${{ matrix.php }} echo "UPX_CMD=$(php src/globals/test-extensions.php upx)" >> $GITHUB_ENV - name: "Run Build Tests (download)" diff --git a/bin/spc-alpine-docker b/bin/spc-alpine-docker index 28cc4939..e2e28dcd 100755 --- a/bin/spc-alpine-docker +++ b/bin/spc-alpine-docker @@ -95,7 +95,7 @@ WORKDIR /app ADD ./src /app/src COPY ./composer.* /app/ ADD ./bin /app/bin -RUN composer install --no-dev --classmap-authoritative +RUN composer install --no-dev EOF fi diff --git a/bin/spc-gnu-docker b/bin/spc-gnu-docker index fa84e58c..101994e4 100755 --- a/bin/spc-gnu-docker +++ b/bin/spc-gnu-docker @@ -145,4 +145,8 @@ echo 'SPC_CMD_VAR_PHP_MAKE_EXTRA_LIBS="-ldl -lpthread -lm -lresolv -lutil -lrt"' # shellcheck disable=SC2086 # shellcheck disable=SC2090 -$DOCKER_EXECUTABLE run --rm $INTERACT -e SPC_FIX_DEPLOY_ROOT="$(pwd)" --env-file /tmp/spc-gnu-docker.env $MOUNT_LIST cwcc-spc-gnu-$SPC_USE_ARCH bin/spc $@ +if [ "$SPC_DOCKER_DEBUG" = "yes" ]; then + $DOCKER_EXECUTABLE run --rm -it --privileged $INTERACT -e SPC_FIX_DEPLOY_ROOT="$(pwd)" --env-file /tmp/spc-gnu-docker.env $MOUNT_LIST cwcc-spc-gnu-$SPC_USE_ARCH +else + $DOCKER_EXECUTABLE run --rm $INTERACT -e SPC_FIX_DEPLOY_ROOT="$(pwd)" --env-file /tmp/spc-gnu-docker.env $MOUNT_LIST cwcc-spc-gnu-$SPC_USE_ARCH bin/spc $@ +fi diff --git a/config/pre-built.json b/config/pre-built.json index 396c8820..22991a61 100644 --- a/config/pre-built.json +++ b/config/pre-built.json @@ -1,6 +1,6 @@ { "repo": "static-php/static-php-cli-hosted", "prefer-stable": true, - "match-pattern": "{name}-{arch}-{os}.txz", - "suffix": "txz" -} \ No newline at end of file + "match-pattern-linux": "{name}-{arch}-{os}-{libc}-{libcver}.txz", + "match-pattern": "{name}-{arch}-{os}.txz" +} diff --git a/src/SPC/builder/LibraryBase.php b/src/SPC/builder/LibraryBase.php index d76aa209..2c75f4ca 100644 --- a/src/SPC/builder/LibraryBase.php +++ b/src/SPC/builder/LibraryBase.php @@ -8,6 +8,7 @@ use SPC\exception\FileSystemException; use SPC\exception\RuntimeException; use SPC\exception\WrongUsageException; use SPC\store\Config; +use SPC\store\Downloader; use SPC\store\FileSystem; use SPC\store\SourceManager; @@ -45,8 +46,9 @@ abstract class LibraryBase $lock = json_decode(FileSystem::readFile(DOWNLOAD_PATH . '/.lock.json'), true) ?? []; $source = Config::getLib(static::NAME, 'source'); // if source is locked as pre-built, we just tryInstall it - if (isset($lock[$source]) && ($lock[$source]['lock_as'] ?? SPC_LOCK_SOURCE) === SPC_LOCK_PRE_BUILT) { - return $this->tryInstall($lock[$source]['filename'], $force); + $pre_built_name = Downloader::getPreBuiltLockName($source); + if (isset($lock[$pre_built_name]) && ($lock[$pre_built_name]['lock_as'] ?? SPC_DOWNLOAD_SOURCE) === SPC_DOWNLOAD_PRE_BUILT) { + return $this->tryInstall($lock[$pre_built_name]['filename'], $force); } return $this->tryBuild($force); } diff --git a/src/SPC/builder/linux/SystemUtil.php b/src/SPC/builder/linux/SystemUtil.php index c0ae6766..e49830d2 100644 --- a/src/SPC/builder/linux/SystemUtil.php +++ b/src/SPC/builder/linux/SystemUtil.php @@ -182,4 +182,39 @@ class SystemUtil 'arch', 'manjaro', ]; } + + /** + * Get libc version string from ldd + */ + public static function getLibcVersionIfExists(): ?string + { + if (PHP_OS_FAMILY === 'Linux' && getenv('SPC_LIBC') === 'glibc') { + $result = shell()->execWithResult('ldd --version', false); + if ($result[0] !== 0) { + return null; + } + // get first line + $first_line = $result[1][0]; + // match ldd version: "ldd (some useless text) 2.17" match 2.17 + $pattern = '/ldd\s+\(.*?\)\s+(\d+\.\d+)/'; + if (preg_match($pattern, $first_line, $matches)) { + return $matches[1]; + } + return null; + } + if (PHP_OS_FAMILY === 'Linux' && getenv('SPC_LIBC') === 'musl') { + if (self::isMuslDist()) { + $result = shell()->execWithResult('ldd 2>&1', false); + } else { + $result = shell()->execWithResult('/usr/local/musl/lib/libc.so 2>&1', false); + } + // Match Version * line + // match ldd version: "Version 1.2.3" match 1.2.3 + $pattern = '/Version\s+(\d+\.\d+\.\d+)/'; + if (preg_match($pattern, $result[1][1] ?? '', $matches)) { + return $matches[1]; + } + } + return null; + } } diff --git a/src/SPC/command/DeleteDownloadCommand.php b/src/SPC/command/DeleteDownloadCommand.php index a953fbe6..b68b8618 100644 --- a/src/SPC/command/DeleteDownloadCommand.php +++ b/src/SPC/command/DeleteDownloadCommand.php @@ -7,6 +7,7 @@ namespace SPC\command; use SPC\exception\DownloaderException; use SPC\exception\FileSystemException; use SPC\exception\WrongUsageException; +use SPC\store\Downloader; use SPC\store\FileSystem; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Input\InputArgument; @@ -47,30 +48,35 @@ class DeleteDownloadCommand extends BaseCommand $chosen_sources = $sources; $lock = json_decode(FileSystem::readFile(DOWNLOAD_PATH . '/.lock.json'), true) ?? []; + $deleted_sources = []; foreach ($chosen_sources as $source) { $source = trim($source); - if (!isset($lock[$source])) { - logger()->warning("Source/Package [{$source}] not locked or not downloaded, skipped."); - continue; + foreach ([$source, Downloader::getPreBuiltLockName($source)] as $name) { + if (isset($lock[$name])) { + $deleted_sources[] = $name; + } } + } + + foreach ($deleted_sources as $lock_name) { // remove download file/dir if exists - if ($lock[$source]['source_type'] === 'archive') { - if (file_exists($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$source]['filename']))) { + if ($lock[$lock_name]['source_type'] === 'archive') { + if (file_exists($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$lock_name]['filename']))) { logger()->info('Deleting file ' . $path); unlink($path); } else { - logger()->warning("Source/Package [{$source}] file not found, skip deleting file."); + logger()->warning("Source/Package [{$lock_name}] file not found, skip deleting file."); } } else { - if (is_dir($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$source]['dirname']))) { + if (is_dir($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$lock_name]['dirname']))) { logger()->info('Deleting dir ' . $path); FileSystem::removeDir($path); } else { - logger()->warning("Source/Package [{$source}] directory not found, skip deleting dir."); + logger()->warning("Source/Package [{$lock_name}] directory not found, skip deleting dir."); } } // remove locked sources - unset($lock[$source]); + unset($lock[$lock_name]); } FileSystem::writeFile(DOWNLOAD_PATH . '/.lock.json', json_encode($lock, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE)); logger()->info('Delete success!'); diff --git a/src/SPC/command/DownloadCommand.php b/src/SPC/command/DownloadCommand.php index 7bf82ebb..b31db2d5 100644 --- a/src/SPC/command/DownloadCommand.php +++ b/src/SPC/command/DownloadCommand.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace SPC\command; +use SPC\builder\linux\SystemUtil; use SPC\builder\traits\UnixSystemUtilTrait; use SPC\exception\DownloaderException; use SPC\exception\FileSystemException; @@ -212,7 +213,7 @@ class DownloadCommand extends BaseCommand if (isset($config['filename'])) { $new_config['filename'] = $config['filename']; } - logger()->info("Fetching source {$source} from custom url [{$ni}/{$cnt}]"); + logger()->info("[{$ni}/{$cnt}] Downloading source {$source} from custom url: {$new_config['url']}"); Downloader::downloadSource($source, $new_config, true); } elseif (isset($custom_gits[$source])) { $config = Config::getSource($source); @@ -224,23 +225,30 @@ class DownloadCommand extends BaseCommand if (isset($config['path'])) { $new_config['path'] = $config['path']; } - logger()->info("Fetching source {$source} from custom git [{$ni}/{$cnt}]"); + logger()->info("[{$ni}/{$cnt}] Downloading source {$source} from custom git: {$new_config['url']}"); Downloader::downloadSource($source, $new_config, true); } else { $config = Config::getSource($source); // Prefer pre-built, we need to search pre-built library if ($this->getOption('prefer-pre-built') && ($config['provide-pre-built'] ?? false) === true) { // We need to replace pattern - $find = str_replace(['{name}', '{arch}', '{os}'], [$source, arch2gnu(php_uname('m')), strtolower(PHP_OS_FAMILY)], Config::getPreBuilt('match-pattern')); + $replace = [ + '{name}' => $source, + '{arch}' => arch2gnu(php_uname('m')), + '{os}' => strtolower(PHP_OS_FAMILY), + '{libc}' => getenv('SPC_LIBC') ?: 'default', + '{libcver}' => PHP_OS_FAMILY === 'Linux' ? (SystemUtil::getLibcVersionIfExists() ?? 'default') : 'default', + ]; + $find = str_replace(array_keys($replace), array_values($replace), Config::getPreBuilt('match-pattern')); // find filename in asset list if (($url = $this->findPreBuilt($pre_built_libs, $find)) !== null) { - logger()->info("Fetching pre-built content {$source} [{$ni}/{$cnt}]"); - Downloader::downloadSource($source, ['type' => 'url', 'url' => $url], $force_all || in_array($source, $force_list), SPC_LOCK_PRE_BUILT); + logger()->info("[{$ni}/{$cnt}] Downloading pre-built content {$source}"); + Downloader::downloadSource($source, ['type' => 'url', 'url' => $url], $force_all || in_array($source, $force_list), SPC_DOWNLOAD_PRE_BUILT); continue; } logger()->warning("Pre-built content not found for {$source}, fallback to source download"); } - logger()->info("Fetching source {$source} [{$ni}/{$cnt}]"); + logger()->info("[{$ni}/{$cnt}] Downloading source {$source}"); Downloader::downloadSource($source, $config, $force_all || in_array($source, $force_list)); } } @@ -352,6 +360,7 @@ class DownloadCommand extends BaseCommand */ private function findPreBuilt(array $assets, string $filename): ?string { + logger()->debug("Finding pre-built asset {$filename}"); foreach ($assets as $asset) { if ($asset['name'] === $filename) { return $asset['browser_download_url']; diff --git a/src/SPC/command/dev/PackLibCommand.php b/src/SPC/command/dev/PackLibCommand.php index 18e02fda..d7eb750a 100644 --- a/src/SPC/command/dev/PackLibCommand.php +++ b/src/SPC/command/dev/PackLibCommand.php @@ -6,6 +6,7 @@ namespace SPC\command\dev; use SPC\builder\BuilderProvider; use SPC\builder\LibraryBase; +use SPC\builder\linux\SystemUtil; use SPC\command\BuildCommand; use SPC\exception\ExceptionHandler; use SPC\exception\FileSystemException; @@ -23,6 +24,7 @@ class PackLibCommand extends BuildCommand public function configure(): void { $this->addArgument('library', InputArgument::REQUIRED, 'The library will be compiled'); + $this->addOption('show-libc-ver', null, null); } public function handle(): int @@ -47,7 +49,7 @@ class PackLibCommand extends BuildCommand // Get lock info $lock = json_decode(file_get_contents(DOWNLOAD_PATH . '/.lock.json'), true) ?? []; $source = Config::getLib($lib->getName(), 'source'); - if (!isset($lock[$source]) || ($lock[$source]['lock_as'] ?? SPC_LOCK_SOURCE) === SPC_LOCK_PRE_BUILT) { + if (!isset($lock[$source]) || ($lock[$source]['lock_as'] ?? SPC_DOWNLOAD_SOURCE) === SPC_DOWNLOAD_PRE_BUILT) { logger()->critical("The library {$lib->getName()} is downloaded as pre-built, we need to build it instead of installing pre-built."); return static::FAILURE; } @@ -69,7 +71,16 @@ class PackLibCommand extends BuildCommand // write list to packlib_files.txt FileSystem::writeFile(WORKING_DIR . '/packlib_files.txt', implode("\n", $increase_files)); // pack - $filename = WORKING_DIR . '/dist/' . $lib->getName() . '-' . arch2gnu(php_uname('m')) . '-' . strtolower(PHP_OS_FAMILY) . '.' . Config::getPreBuilt('suffix'); + $filename = Config::getPreBuilt('match-pattern'); + $replace = [ + '{name}' => $lib->getName(), + '{arch}' => arch2gnu(php_uname('m')), + '{os}' => strtolower(PHP_OS_FAMILY), + '{libc}' => getenv('SPC_LIBC') ?: 'default', + '{libcver}' => PHP_OS_FAMILY === 'Linux' ? (SystemUtil::getLibcVersionIfExists() ?? 'default') : 'default', + ]; + $filename = str_replace(array_keys($replace), array_values($replace), $filename); + $filename = WORKING_DIR . '/dist/' . $filename; f_passthru('tar -czf ' . $filename . ' -T ' . WORKING_DIR . '/packlib_files.txt'); logger()->info('Pack library ' . $lib->getName() . ' to ' . $filename . ' complete.'); } diff --git a/src/SPC/store/Config.php b/src/SPC/store/Config.php index 07ed2887..68ba4399 100644 --- a/src/SPC/store/Config.php +++ b/src/SPC/store/Config.php @@ -22,11 +22,30 @@ class Config public static ?array $pre_built = null; + /** + * @throws WrongUsageException + * @throws FileSystemException + */ public static function getPreBuilt(string $name): mixed { if (self::$pre_built === null) { self::$pre_built = FileSystem::loadConfigArray('pre-built'); } + $supported_sys_based = ['match-pattern', 'prefer-stable', 'repo']; + if (in_array($name, $supported_sys_based)) { + $m_key = match (PHP_OS_FAMILY) { + 'Windows' => ['-windows', '-win', ''], + 'Darwin' => ['-macos', '-unix', ''], + 'Linux' => ['-linux', '-unix', ''], + 'BSD' => ['-freebsd', '-bsd', '-unix', ''], + default => throw new WrongUsageException('OS ' . PHP_OS_FAMILY . ' is not supported'), + }; + foreach ($m_key as $v) { + if (isset(self::$pre_built["{$name}{$v}"])) { + return self::$pre_built["{$name}{$v}"]; + } + } + } return self::$pre_built[$name] ?? null; } diff --git a/src/SPC/store/Downloader.php b/src/SPC/store/Downloader.php index fbaeb7a7..7a857c48 100644 --- a/src/SPC/store/Downloader.php +++ b/src/SPC/store/Downloader.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace SPC\store; +use SPC\builder\linux\SystemUtil; use SPC\exception\DownloaderException; use SPC\exception\FileSystemException; use SPC\exception\RuntimeException; @@ -124,6 +125,7 @@ class Downloader if (($source['prefer-stable'] ?? false) === true && $release['prerelease'] === true) { continue; } + logger()->debug("Found {$release['name']} releases assets"); if (!$match_result) { return $release['assets']; } @@ -189,7 +191,7 @@ class Downloader * @throws RuntimeException * @throws WrongUsageException */ - public static function downloadFile(string $name, string $url, string $filename, ?string $move_path = null, int $lock_as = SPC_LOCK_SOURCE): void + public static function downloadFile(string $name, string $url, string $filename, ?string $move_path = null, int $download_as = SPC_DOWNLOAD_SOURCE): void { logger()->debug("Downloading {$url}"); $cancel_func = function () use ($filename) { @@ -202,12 +204,23 @@ class Downloader self::curlDown(url: $url, path: FileSystem::convertPath(DOWNLOAD_PATH . "/{$filename}"), retry: self::getRetryTime()); self::unregisterCancelEvent(); logger()->debug("Locking {$filename}"); - self::lockSource($name, ['source_type' => 'archive', 'filename' => $filename, 'move_path' => $move_path, 'lock_as' => $lock_as]); + if ($download_as === SPC_DOWNLOAD_PRE_BUILT) { + $name = self::getPreBuiltLockName($name); + } + self::lockSource($name, ['source_type' => 'archive', 'filename' => $filename, 'move_path' => $move_path, 'lock_as' => $download_as]); } /** * Try to lock source. * + * @param string $name Source name + * @param array{ + * source_type: string, + * dirname: ?string, + * filename: ?string, + * move_path: ?string, + * lock_as: int + * } $data Source data * @throws FileSystemException */ public static function lockSource(string $name, array $data): void @@ -228,7 +241,7 @@ class Downloader * @throws RuntimeException * @throws WrongUsageException */ - public static function downloadGit(string $name, string $url, string $branch, ?string $move_path = null, int $retry = 0, int $lock_as = SPC_LOCK_SOURCE): void + public static function downloadGit(string $name, string $url, string $branch, ?string $move_path = null, int $retry = 0, int $lock_as = SPC_DOWNLOAD_SOURCE): void { $download_path = FileSystem::convertPath(DOWNLOAD_PATH . "/{$name}"); if (file_exists($download_path)) { @@ -246,6 +259,7 @@ class Downloader self::registerCancelEvent($cancel_func); f_passthru( SPC_GIT_EXEC . ' clone' . $check . + (defined('DEBUG_MODE') ? '' : ' --quiet') . ' --config core.autocrlf=false ' . "--branch \"{$branch}\" " . (defined('GIT_SHALLOW_CLONE') ? '--depth 1 --single-branch' : '') . " --recursive \"{$url}\" \"{$download_path}\"" ); @@ -283,8 +297,22 @@ class Downloader } /** + * @param string $name Package name + * @param null|array{ + * type: string, + * repo: ?string, + * url: ?string, + * rev: ?string, + * path: ?string, + * filename: ?string, + * match: ?string, + * prefer-stable: ?bool, + * extract-files: ?array + * } $pkg Package config + * @param bool $force Download all the time even if it exists * @throws DownloaderException * @throws FileSystemException + * @throws WrongUsageException */ public static function downloadPackage(string $name, ?array $pkg = null, bool $force = false): void { @@ -301,50 +329,36 @@ class Downloader FileSystem::createDir(DOWNLOAD_PATH); } - // load lock file - if (!file_exists(DOWNLOAD_PATH . '/.lock.json')) { - $lock = []; - } else { - $lock = json_decode(FileSystem::readFile(DOWNLOAD_PATH . '/.lock.json'), true) ?? []; - } - // If lock file exists, skip downloading - if (isset($lock[$name]) && !$force) { - if ($lock[$name]['source_type'] === 'archive' && file_exists(DOWNLOAD_PATH . '/' . $lock[$name]['filename'])) { - logger()->notice("Package [{$name}] already downloaded: " . $lock[$name]['filename']); - return; - } - if ($lock[$name]['source_type'] === 'dir' && is_dir(DOWNLOAD_PATH . '/' . $lock[$name]['dirname'])) { - logger()->notice("Package [{$name}] already downloaded: " . $lock[$name]['dirname']); - return; - } + if (self::isAlreadyDownloaded($name, $force, SPC_DOWNLOAD_PACKAGE)) { + return; } try { switch ($pkg['type']) { case 'bitbuckettag': // BitBucket Tag [$url, $filename] = self::getLatestBitbucketTag($name, $pkg); - self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_LOCK_PRE_BUILT); + self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_DOWNLOAD_PACKAGE); break; case 'ghtar': // GitHub Release (tar) [$url, $filename] = self::getLatestGithubTarball($name, $pkg); - self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_LOCK_PRE_BUILT); + self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_DOWNLOAD_PACKAGE); break; case 'ghtagtar': // GitHub Tag (tar) [$url, $filename] = self::getLatestGithubTarball($name, $pkg, 'tags'); - self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_LOCK_PRE_BUILT); + self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_DOWNLOAD_PACKAGE); break; case 'ghrel': // GitHub Release (uploaded) [$url, $filename] = self::getLatestGithubRelease($name, $pkg); - self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_LOCK_PRE_BUILT); + self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_DOWNLOAD_PACKAGE); break; case 'filelist': // Basic File List (regex based crawler) [$url, $filename] = self::getFromFileList($name, $pkg); - self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_LOCK_PRE_BUILT); + self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_DOWNLOAD_PACKAGE); break; case 'url': // Direct download URL $url = $pkg['url']; $filename = $pkg['filename'] ?? basename($pkg['url']); - self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_LOCK_PRE_BUILT); + self::downloadFile($name, $url, $filename, $pkg['extract'] ?? null, SPC_DOWNLOAD_PACKAGE); break; case 'git': // Git repo self::downloadGit( @@ -353,7 +367,7 @@ class Downloader $pkg['rev'], $pkg['extract'] ?? null, self::getRetryTime(), - SPC_LOCK_PRE_BUILT + SPC_DOWNLOAD_PRE_BUILT ); break; case 'custom': // Custom download method, like API-based download or other @@ -382,15 +396,30 @@ class Downloader /** * Download source by name and meta. * - * @param string $name source name - * @param null|array $source source meta info: [type, path, rev, url, filename, regex, license] - * @param bool $force Whether to force download (default: false) - * @param int $lock_as Lock source type (default: SPC_LOCK_SOURCE) + * @param string $name source name + * @param null|array{ + * type: string, + * repo: ?string, + * url: ?string, + * rev: ?string, + * path: ?string, + * filename: ?string, + * match: ?string, + * prefer-stable: ?bool, + * provide-pre-built: ?bool, + * license: array{ + * type: string, + * path: ?string, + * text: ?string + * } + * } $source source meta info: [type, path, rev, url, filename, regex, license] + * @param bool $force Whether to force download (default: false) + * @param int $download_as Lock source type (default: SPC_LOCK_SOURCE) * @throws DownloaderException * @throws FileSystemException * @throws WrongUsageException */ - public static function downloadSource(string $name, ?array $source = null, bool $force = false, int $lock_as = SPC_LOCK_SOURCE): void + public static function downloadSource(string $name, ?array $source = null, bool $force = false, int $download_as = SPC_DOWNLOAD_SOURCE): void { if ($source === null) { $source = Config::getSource($name); @@ -406,49 +435,36 @@ class Downloader } // load lock file - if (!file_exists(DOWNLOAD_PATH . '/.lock.json')) { - $lock = []; - } else { - $lock = json_decode(FileSystem::readFile(DOWNLOAD_PATH . '/.lock.json'), true) ?? []; - } - // If lock file exists, skip downloading - if (isset($lock[$name]) && !$force && ($lock[$name]['lock_as'] ?? SPC_LOCK_SOURCE) === $lock_as) { - if ($lock[$name]['source_type'] === 'archive' && file_exists(DOWNLOAD_PATH . '/' . $lock[$name]['filename'])) { - logger()->notice("source [{$name}] already downloaded: " . $lock[$name]['filename']); - return; - } - if ($lock[$name]['source_type'] === 'dir' && is_dir(DOWNLOAD_PATH . '/' . $lock[$name]['dirname'])) { - logger()->notice("source [{$name}] already downloaded: " . $lock[$name]['dirname']); - return; - } + if (self::isAlreadyDownloaded($name, $force, $download_as)) { + return; } try { switch ($source['type']) { case 'bitbuckettag': // BitBucket Tag [$url, $filename] = self::getLatestBitbucketTag($name, $source); - self::downloadFile($name, $url, $filename, $source['path'] ?? null, $lock_as); + self::downloadFile($name, $url, $filename, $source['path'] ?? null, $download_as); break; case 'ghtar': // GitHub Release (tar) [$url, $filename] = self::getLatestGithubTarball($name, $source); - self::downloadFile($name, $url, $filename, $source['path'] ?? null, $lock_as); + self::downloadFile($name, $url, $filename, $source['path'] ?? null, $download_as); break; case 'ghtagtar': // GitHub Tag (tar) [$url, $filename] = self::getLatestGithubTarball($name, $source, 'tags'); - self::downloadFile($name, $url, $filename, $source['path'] ?? null, $lock_as); + self::downloadFile($name, $url, $filename, $source['path'] ?? null, $download_as); break; case 'ghrel': // GitHub Release (uploaded) [$url, $filename] = self::getLatestGithubRelease($name, $source); - self::downloadFile($name, $url, $filename, $source['path'] ?? null, $lock_as); + self::downloadFile($name, $url, $filename, $source['path'] ?? null, $download_as); break; case 'filelist': // Basic File List (regex based crawler) [$url, $filename] = self::getFromFileList($name, $source); - self::downloadFile($name, $url, $filename, $source['path'] ?? null, $lock_as); + self::downloadFile($name, $url, $filename, $source['path'] ?? null, $download_as); break; case 'url': // Direct download URL $url = $source['url']; $filename = $source['filename'] ?? basename($source['url']); - self::downloadFile($name, $url, $filename, $source['path'] ?? null, $lock_as); + self::downloadFile($name, $url, $filename, $source['path'] ?? null, $download_as); break; case 'git': // Git repo self::downloadGit( @@ -457,14 +473,14 @@ class Downloader $source['rev'], $source['path'] ?? null, self::getRetryTime(), - $lock_as + $download_as ); break; case 'custom': // Custom download method, like API-based download or other $classes = FileSystem::getClassesPsr4(ROOT_DIR . '/src/SPC/store/source', 'SPC\store\source'); foreach ($classes as $class) { if (is_a($class, CustomSourceBase::class, true) && $class::NAME === $name) { - (new $class())->fetch($force, $source, $lock_as); + (new $class())->fetch($force, $source, $download_as); break; } } @@ -579,6 +595,11 @@ class Downloader } } + public static function getPreBuiltLockName(string $source): string + { + return "{$source}-" . PHP_OS_FAMILY . '-' . getenv('GNU_ARCH') . '-' . (getenv('SPC_LIBC') ?: 'default') . '-' . (SystemUtil::getLibcVersionIfExists() ?? 'default'); + } + /** * Register CTRL+C event for different OS. * @@ -611,4 +632,39 @@ class Downloader { return intval(getenv('SPC_RETRY_TIME') ? getenv('SPC_RETRY_TIME') : 0); } + + /** + * @throws FileSystemException + */ + private static function isAlreadyDownloaded(string $name, bool $force, int $download_as = SPC_DOWNLOAD_SOURCE): bool + { + if (!file_exists(DOWNLOAD_PATH . '/.lock.json')) { + $lock = []; + } else { + $lock = json_decode(FileSystem::readFile(DOWNLOAD_PATH . '/.lock.json'), true) ?? []; + } + // If lock file exists, skip downloading for source mode + if (!$force && $download_as === SPC_DOWNLOAD_SOURCE && isset($lock[$name])) { + if ( + $lock[$name]['source_type'] === 'archive' && file_exists(DOWNLOAD_PATH . '/' . $lock[$name]['filename']) || + $lock[$name]['source_type'] === 'dir' && is_dir(DOWNLOAD_PATH . '/' . $lock[$name]['dirname']) + ) { + logger()->notice("Source [{$name}] already downloaded: " . ($lock[$name]['filename'] ?? $lock[$name]['dirname'])); + return true; + } + } + // If lock file exists for current arch and glibc target, skip downloading + + if (!$force && $download_as === SPC_DOWNLOAD_PRE_BUILT && isset($lock[$lock_name = self::getPreBuiltLockName($name)])) { + // lock name with env + if ( + $lock[$lock_name]['source_type'] === 'archive' && file_exists(DOWNLOAD_PATH . '/' . $lock[$lock_name]['filename']) || + $lock[$lock_name]['source_type'] === 'dir' && is_dir(DOWNLOAD_PATH . '/' . $lock[$lock_name]['dirname']) + ) { + logger()->notice("Pre-built content [{$name}] already downloaded: " . ($lock[$lock_name]['filename'] ?? $lock[$lock_name]['dirname'])); + return true; + } + } + return false; + } } diff --git a/src/SPC/store/SourceManager.php b/src/SPC/store/SourceManager.php index 78f68ad4..c36bfc58 100644 --- a/src/SPC/store/SourceManager.php +++ b/src/SPC/store/SourceManager.php @@ -54,15 +54,22 @@ class SourceManager if (Config::getSource($source) === null) { throw new WrongUsageException("Source [{$source}] does not exist, please check the name and correct it !"); } - if (!isset($lock[$source])) { - throw new WrongUsageException('Source [' . $source . '] not downloaded or not locked, you should download it first !'); + // check source downloaded + $pre_built_name = Downloader::getPreBuiltLockName($source); + if (!isset($lock[$pre_built_name])) { + if (!isset($lock[$source])) { + throw new WrongUsageException("Source [{$source}] not downloaded or not locked, you should download it first !"); + } + $lock_name = $source; + } else { + $lock_name = $pre_built_name; } // check source dir exist - $check = $lock[$source]['move_path'] === null ? (SOURCE_PATH . '/' . $source) : (SOURCE_PATH . '/' . $lock[$source]['move_path']); + $check = $lock[$lock_name]['move_path'] === null ? (SOURCE_PATH . '/' . $source) : (SOURCE_PATH . '/' . $lock[$lock_name]['move_path']); if (!is_dir($check)) { logger()->debug('Extracting source [' . $source . '] to ' . $check . ' ...'); - FileSystem::extractSource($source, DOWNLOAD_PATH . '/' . ($lock[$source]['filename'] ?? $lock[$source]['dirname']), $lock[$source]['move_path']); + FileSystem::extractSource($source, DOWNLOAD_PATH . '/' . ($lock[$lock_name]['filename'] ?? $lock[$lock_name]['dirname']), $lock[$lock_name]['move_path']); } else { logger()->debug('Source [' . $source . '] already extracted in ' . $check . ', skip !'); } diff --git a/src/SPC/store/source/CustomSourceBase.php b/src/SPC/store/source/CustomSourceBase.php index 2df8eb7e..3fce1568 100644 --- a/src/SPC/store/source/CustomSourceBase.php +++ b/src/SPC/store/source/CustomSourceBase.php @@ -8,5 +8,5 @@ abstract class CustomSourceBase { public const NAME = 'unknown'; - abstract public function fetch(bool $force = false, ?array $config = null, int $lock_as = SPC_LOCK_SOURCE): void; + abstract public function fetch(bool $force = false, ?array $config = null, int $lock_as = SPC_DOWNLOAD_SOURCE): void; } diff --git a/src/SPC/store/source/PhpSource.php b/src/SPC/store/source/PhpSource.php index d15fd40f..d2617e3b 100644 --- a/src/SPC/store/source/PhpSource.php +++ b/src/SPC/store/source/PhpSource.php @@ -17,7 +17,7 @@ class PhpSource extends CustomSourceBase * @throws DownloaderException * @throws FileSystemException */ - public function fetch(bool $force = false, ?array $config = null, int $lock_as = SPC_LOCK_SOURCE): void + public function fetch(bool $force = false, ?array $config = null, int $lock_as = SPC_DOWNLOAD_SOURCE): void { $major = defined('SPC_BUILD_PHP_VERSION') ? SPC_BUILD_PHP_VERSION : '8.3'; Downloader::downloadSource('php-src', self::getLatestPHPInfo($major), $force); diff --git a/src/SPC/store/source/PostgreSQLSource.php b/src/SPC/store/source/PostgreSQLSource.php index 113abc98..7c85a8d4 100644 --- a/src/SPC/store/source/PostgreSQLSource.php +++ b/src/SPC/store/source/PostgreSQLSource.php @@ -16,7 +16,7 @@ class PostgreSQLSource extends CustomSourceBase * @throws DownloaderException * @throws FileSystemException */ - public function fetch(bool $force = false, ?array $config = null, int $lock_as = SPC_LOCK_SOURCE): void + public function fetch(bool $force = false, ?array $config = null, int $lock_as = SPC_DOWNLOAD_SOURCE): void { Downloader::downloadSource('postgresql', self::getLatestInfo(), $force); } diff --git a/src/globals/defines.php b/src/globals/defines.php index 46c98915..699cecc1 100644 --- a/src/globals/defines.php +++ b/src/globals/defines.php @@ -41,8 +41,9 @@ const SPC_EXTENSION_ALIAS = [ ]; // spc lock type -const SPC_LOCK_SOURCE = 1; // lock source -const SPC_LOCK_PRE_BUILT = 2; // lock pre-built +const SPC_DOWNLOAD_SOURCE = 1; // lock source +const SPC_DOWNLOAD_PRE_BUILT = 2; // lock pre-built +const SPC_DOWNLOAD_PACKAGE = 3; // lock as package // file replace strategy const REPLACE_FILE_STR = 1; diff --git a/src/globals/test-extensions.php b/src/globals/test-extensions.php index cecb8ace..c8e2f8cb 100644 --- a/src/globals/test-extensions.php +++ b/src/globals/test-extensions.php @@ -13,9 +13,9 @@ declare(strict_types=1); // test php version (8.1 ~ 8.4 available, multiple for matrix) $test_php_version = [ - '8.1', - '8.2', - '8.3', + // '8.1', + // '8.2', + // '8.3', '8.4', ]; @@ -24,7 +24,9 @@ $test_os = [ // 'macos-13', // 'macos-14', 'ubuntu-latest', - 'windows-latest', + 'ubuntu-22.04', + 'ubuntu-22.04-arm', + 'ubuntu-24.04-arm', ]; // whether enable thread safe @@ -33,14 +35,14 @@ $zts = false; $no_strip = false; // compress with upx -$upx = false; +$upx = true; // prefer downloading pre-built packages to speed up the build process -$prefer_pre_built = false; +$prefer_pre_built = true; // If you want to test your added extensions and libs, add below (comma separated, example `bcmath,openssl`). $extensions = match (PHP_OS_FAMILY) { - 'Linux', 'Darwin' => 'pgsql,pdo_pgsql', + 'Linux', 'Darwin' => 'imagick', 'Windows' => 'pgsql,pdo_pgsql', }; @@ -115,6 +117,13 @@ if ($argv[1] === 'download_cmd') { $down_cmd .= $prefer_pre_built ? '--prefer-pre-built ' : ''; } +if ($argv[1] === 'doctor_cmd') { + $doctor_cmd = 'doctor --auto-fix --debug'; +} +if ($argv[1] === 'install_upx_cmd') { + $install_upx_cmd = 'install-pkg upx'; +} + // generate build command if ($argv[1] === 'build_cmd' || $argv[1] === 'build_embed_cmd') { $build_cmd = 'build '; @@ -139,30 +148,36 @@ echo match ($argv[1]) { 'upx' => $upx ? '--with-upx-pack' : '', 'prefer_pre_built' => $prefer_pre_built ? '--prefer-pre-built' : '', 'download_cmd' => $down_cmd, + 'install_upx_cmd' => $install_upx_cmd, + 'doctor_cmd' => $doctor_cmd, 'build_cmd' => $build_cmd, 'build_embed_cmd' => $build_cmd, default => '', }; +$prefix = match ($argv[2] ?? null) { + 'windows-latest', 'windows-2022', 'windows-2019', 'windows-2025' => 'powershell.exe -file .\bin\spc.ps1 ', + 'ubuntu-latest', 'ubuntu-24.04', 'ubuntu-24.04-arm' => './bin/spc ', + 'ubuntu-22.04', 'ubuntu-22.04-arm' => 'bin/spc-gnu-docker ', + 'ubuntu-20.04' => 'bin/spc-alpine-docker ', + default => 'bin/spc ', +}; + if ($argv[1] === 'download_cmd') { - if (str_starts_with($argv[2], 'windows-')) { - passthru('powershell.exe -file .\bin\spc.ps1 ' . $down_cmd, $retcode); - } else { - passthru('./bin/spc ' . $down_cmd, $retcode); - } + passthru($prefix . $down_cmd, $retcode); } elseif ($argv[1] === 'build_cmd') { - if (str_starts_with($argv[2], 'windows-')) { - passthru('powershell.exe -file .\bin\spc.ps1 ' . $build_cmd . ' --build-cli --build-micro', $retcode); - } else { - passthru('./bin/spc ' . $build_cmd . ' --build-cli --build-micro', $retcode); - } + passthru($prefix . $build_cmd . ' --build-cli --build-micro', $retcode); } elseif ($argv[1] === 'build_embed_cmd') { if (str_starts_with($argv[2], 'windows-')) { // windows does not accept embed SAPI - passthru('powershell.exe -file .\bin\spc.ps1 ' . $build_cmd . ' --build-cli', $retcode); + passthru($prefix . $build_cmd . ' --build-cli', $retcode); } else { - passthru('./bin/spc ' . $build_cmd . ' --build-embed', $retcode); + passthru($prefix . $build_cmd . ' --build-embed', $retcode); } +} elseif ($argv[1] === 'doctor_cmd') { + passthru($prefix . $doctor_cmd, $retcode); +} elseif ($argv[1] === 'install_upx_cmd') { + passthru($prefix . $install_upx_cmd, $retcode); } else { $retcode = 0; }