diff --git a/.github/.cspell/organization-dictionary.txt b/.github/.cspell/organization-dictionary.txt index d7546a20..7244875f 100644 --- a/.github/.cspell/organization-dictionary.txt +++ b/.github/.cspell/organization-dictionary.txt @@ -151,6 +151,7 @@ endianness esac euxo gsub +libc moreutils msys noninteractive diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index fb6b32c6..bc45dc6f 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -6,7 +6,7 @@ dprint enablerepo epel grcov -libc +incompat linkcheck mdbook microdnf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 505806d8..07463c38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,25 +38,19 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-20.04 - - ubuntu-22.04 - tool: - # cargo-watch/watchexec-cli is supported by cargo-binstall (through quickinstall) - # TODO: valgrind installation sometime hangs. - - cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft include: - # Note: Specifying the version of valgrind and cargo-binstall is not supported. - os: ubuntu-20.04 - tool: cargo-dinghy@0.6.4,cargo-hack@0.5.24,cargo-llvm-cov@0.5.3,cargo-make@0.36.11,cargo-minimal-versions@0.1.8,cargo-no-dev-deps@0.1.0,parse-changelog@0.5.2,cargo-udeps@0.1.35,cargo-valgrind@2.1.0,cargo-deny@0.13.5,cross@0.2.4,dprint@0.34.1,just@1.9.0,nextest@0.9.11,protoc@3.21.12,shellcheck@0.9.0,shfmt@3.6.0,wasm-pack@0.10.3,wasmtime@4.0.0,mdbook@0.4.25,mdbook-linkcheck@0.7.7,cargo-watch@8.1.1,grcov@0.8.13,watchexec-cli@1.20.5,cargo-tarpaulin@0.25.0,zola@0.16.1,syft@0.83.0 + - os: ubuntu-22.04 - os: ubuntu-20.04 - tool: cargo-dinghy@0.6,cargo-hack@0.5,cargo-llvm-cov@0.5,cargo-make@0.36,cargo-minimal-versions@0.1,cargo-no-dev-deps@0.1,parse-changelog@0.5,cargo-udeps@0.1,cargo-valgrind@2.1,cargo-deny@0.13,cross@0.2,dprint@0.34,just@1.9,nextest@0.9,protoc@3.21,shellcheck@0.9,shfmt@3.5,wasm-pack@0.10,wasmtime@6.0,mdbook@0.4,mdbook-linkcheck@0.7,cargo-watch@8.1,grcov@0.8,watchexec-cli@1.20,cargo-tarpaulin@0.25,zola@0.16,syft@0.83 + tool: major.minor.patch - os: ubuntu-20.04 - tool: cargo-valgrind@2, just@1,protoc@3 , shfmt@3 ,wasmtime@7,cargo-watch@8,watchexec-cli@1 + tool: major.minor + - os: ubuntu-20.04 + tool: major - os: macos-11 - tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft + - os: macos-12 - os: windows-2019 - tool: cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft + - os: windows-2022 runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -65,9 +59,12 @@ jobs: persist-credentials: false # cross attempts to install rust-src when Cargo.toml is available even if `cross --version` - run: rm Cargo.toml + - name: Generate tool list + id: tool-list + run: tools/ci/tool-list.sh "${{ matrix.tool }}" >>"${GITHUB_OUTPUT}" - uses: ./ with: - tool: ${{ matrix.tool }} + tool: ${{ steps.tool-list.outputs.tool }} # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell - name: Test bash run: just --version; shfmt --version; protoc --version @@ -97,33 +94,17 @@ jobs: fail-fast: false matrix: container: - # glibc >= 2.31 + - ubuntu:18.04 # glibc 2.27 - ubuntu:20.04 # glibc 2.31 - ubuntu:22.04 # glibc 2.35 + - debian:10-slim # glibc 2.28 - debian:11-slim # glibc 2.31 - debian:12-slim # glibc 2.36 - fedora:latest # glibc 2.37 (as of fedora 38) - tool: - # valgrind: installing snap to container is difficult... - - cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,zola,syft - include: - # glibc < 2.31 - # zola don't provide prebuilt binaries for musl or old glibc host. - - container: ubuntu:18.04 # glibc 2.27 - tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft - - container: debian:10-slim # glibc 2.28 - tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft - - container: rockylinux:8 # glibc 2.28 - tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft - - container: rockylinux:8-minimal # glibc 2.28 - tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch,grcov,watchexec-cli,cargo-tarpaulin,syft - # glibc < 2.27 or musl - - container: centos:7 # glibc 2.17 - # protoc,valgrind,wasmtime,mdbook-linkcheck,cargo-watch,zola don't provide prebuilt binaries for musl or old glibc host. - tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,shellcheck,shfmt,wasm-pack,mdbook,cargo-binstall,grcov,watchexec-cli,cargo-tarpaulin,syft - - container: alpine:latest # musl 1.2.4 (as of alpine 3.18) - # protoc,valgrind,wasmtime,mdbook-linkcheck,cargo-watch,zola don't provide prebuilt binaries for musl host. - tool: cargo-dinghy,cargo-hack,cargo-llvm-cov,cargo-make,cargo-minimal-versions,cargo-no-dev-deps,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,dprint,just,nextest,shellcheck,shfmt,wasm-pack,mdbook,cargo-binstall,grcov,watchexec-cli,cargo-tarpaulin,syft + - rockylinux:8 # glibc 2.28 + - rockylinux:8-minimal # glibc 2.28 + - centos:7 # glibc 2.17 + - alpine:latest # musl 1.2.4 (as of alpine 3.18) runs-on: ubuntu-latest timeout-minutes: 60 container: ${{ matrix.container }} @@ -161,9 +142,12 @@ jobs: persist-credentials: false # cross attempts to install rust-src when Cargo.toml is available even if `cross --version` - run: rm Cargo.toml + - name: Generate tool list + id: tool-list + run: tools/ci/tool-list.sh >>"${GITHUB_OUTPUT}" - uses: ./ with: - tool: ${{ matrix.tool }} + tool: ${{ steps.tool-list.outputs.tool }} manifest: runs-on: ubuntu-latest diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f6c1f295..c6cc9771 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -21,13 +21,14 @@ See JSON files in `tools/codegen/base` directory for examples of the manifest. > GITHUB_TOKEN=$(gh auth status --show-token 2>&1 | sed -n 's/^.*Token: \(.*\)$/\1/p') ./tools/manifest.sh > ``` -3\. Add tool name to test matrix in `.github/workflows/ci.yml`. - -4\. Add tool name to the table in ["Supported tools" section in `README.md`](https://github.com/taiki-e/install-action#supported-tools). +3\. Add tool name to the table in ["Supported tools" section in `README.md`](https://github.com/taiki-e/install-action#supported-tools). ## Troubleshooting -If one of the CI builds fails due to an bin path or release asset_name, fix the problem in the base +If one of the CI builds fails due to a bin path or release asset_name, fix the problem in the base manifest, and re-run the manifest tool `tools/manifest.sh` to regenerate the manifest json file. The base manifest supports overriding the bin path per platform by adding the `"bin"` / `"asset_name"` to the platform object. + +If CI fails only for containers using older versions of glibc or musl, you may need to add the tool +name to one of the `*_incompat` arrays in `tools/ci/tool-list.sh`. diff --git a/tools/ci/manifest.sh b/tools/ci/manifest.sh index ac2cb09d..342bfe51 100755 --- a/tools/ci/manifest.sh +++ b/tools/ci/manifest.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 OR MIT -set -euxo pipefail +set -euo pipefail IFS=$'\n\t' cd "$(dirname "$0")"/../.. @@ -16,6 +16,8 @@ fi git config user.name "Taiki Endo" git config user.email "te316e89@gmail.com" +set -x + for manifest in manifests/*.json; do git add -N "${manifest}" if ! git diff --exit-code -- "${manifest}"; then diff --git a/tools/ci/tool-list.sh b/tools/ci/tool-list.sh new file mode 100755 index 00000000..55eb03f8 --- /dev/null +++ b/tools/ci/tool-list.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 OR MIT +set -euo pipefail +IFS=$'\n\t' +cd "$(dirname "$0")"/../.. + +# They don't provide prebuilt binaries for musl or old glibc host. +glibc_pre_2_31_incompat=( + zola +) +glibc_pre_2_27_incompat=( + "${glibc_pre_2_31_incompat[@]}" + cargo-watch + mdbook-linkcheck + protoc + valgrind + wasmtime +) +musl_incompat=( + "${glibc_pre_2_27_incompat[@]}" +) + +incompat_tools=() +case "${1:-}" in + '') version=latest ;; + major.minor.patch | major.minor | major) + version="$1" + # Specifying the version of valgrind and cargo-binstall is not supported. + incompat_tools+=(valgrind cargo-binstall) + ;; + *) + echo "tool=$1" + exit 0 + ;; +esac +case "$(uname -s)" in + Linux) + host_os=linux + ldd_version=$(ldd --version 2>&1 || true) + if grep <<<"${ldd_version}" -q 'musl'; then + incompat_tools+=("${musl_incompat[@]}") + else + host_glibc_version=$(grep <<<"${ldd_version}" -E "GLIBC|GNU libc" | sed "s/.* //g") + higher_glibc_version=$(sort <<<"2.31"$'\n'"${host_glibc_version}" -Vu | tail -1) + if [[ "${higher_glibc_version}" != "${host_glibc_version}" ]]; then + higher_glibc_version=$(sort <<<"2.27"$'\n'"${host_glibc_version}" -Vu | tail -1) + if [[ "${higher_glibc_version}" == "${host_glibc_version}" ]]; then + incompat_tools+=("${glibc_pre_2_31_incompat[@]}") + else + incompat_tools+=("${glibc_pre_2_27_incompat[@]}") + fi + fi + fi + if ! type -P snap &>/dev/null; then + incompat_tools+=(valgrind) + fi + ;; + Darwin) host_os=macos ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) host_os=windows ;; + *) bail "unrecognized OS type '$(uname -s)'" ;; +esac + +tools=() +for manifest in tools/codegen/base/*.json; do + tool_name=$(basename "${manifest%.*}") + case "${host_os}" in + linux*) + for incompat in ${incompat_tools[@]+"${incompat_tools[@]}"}; do + if [[ "${incompat}" == "${tool_name}" ]]; then + tool_name='' + break + fi + done + ;; + *) + if [[ "$(jq -r ".platform.x86_64_${host_os}" "${manifest}")" == "null" ]]; then + continue + fi + ;; + esac + if [[ -n "${tool_name}" ]]; then + if [[ "${version}" != "latest" ]]; then + latest_version="$(jq -r ".latest.version" "manifests/${tool_name}.json")" + case "${version}" in + major.minor.patch) tool_name+="@${latest_version}" ;; + major.minor) tool_name+="@${latest_version%.*}" ;; + major) tool_name+="@${latest_version%%.*}" ;; + *) exit 1 ;; + esac + fi + if [[ "${tool_name}" != *"@0" ]] && [[ "${tool_name}" != *"@0.0" ]]; then + tools+=("${tool_name}") + fi + fi +done +if [[ "${version}" != "latest" ]]; then + tools_tmp=() + for tool in "${tools[@]}"; do + tools_tmp+=("${tool}") + done + tools=("${tools_tmp[@]}") +fi + +# Not manifest-based +case "${host_os}" in + linux*) + # Installing snap to container is difficult... + # Specifying the version of valgrind is not supported. + if type -P snap &>/dev/null && [[ "${version}" == "latest" ]]; then + tools+=(valgrind) + fi + ;; +esac +# cargo-watch/watchexec-cli is supported by cargo-binstall (through quickinstall) +case "${version}" in + latest) tools+=(cargo-watch watchexec-cli) ;; + major.minor.patch) tools+=(cargo-watch@8.1.1 watchexec-cli@1.20.5) ;; + major.minor) tools+=(cargo-watch@8.1 watchexec-cli@1.20) ;; + major) tools+=(cargo-watch@8 watchexec-cli@1) ;; + *) exit 1 ;; +esac + +# sort and dedup +IFS=$'\n' +# shellcheck disable=SC2207 +tools=($(LC_ALL=C sort -u <<<"${tools[*]}")) +IFS=$'\n\t' + +# TODO: inject random space before/after of tool name for testing https://github.com/taiki-e/install-action/issues/115. +IFS=',' +echo "tool=${tools[*]}" +IFS=$'\n\t'