diff --git a/.github/.cspell/project-dictionary.txt b/.github/.cspell/project-dictionary.txt index cfbbf892..f9d94917 100644 --- a/.github/.cspell/project-dictionary.txt +++ b/.github/.cspell/project-dictionary.txt @@ -1,8 +1,11 @@ binstall bytecodealliance +coreutils distro doas Dpkg +enablerepo +epel jfrimmel koalaman libc diff --git a/.github/.cspell/rust-dependencies.txt b/.github/.cspell/rust-dependencies.txt index e69de29b..48f1cbfc 100644 --- a/.github/.cspell/rust-dependencies.txt +++ b/.github/.cspell/rust-dependencies.txt @@ -0,0 +1,8 @@ +// This file is @generated by tidy.sh. +// It is not intended for manual editing. + +anyhow +json +semver +serde +ureq diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a848b973..4edcc0de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ defaults: jobs: tidy: - uses: taiki-e/workflows/.github/workflows/tidy-no-rust.yml@main + uses: taiki-e/workflows/.github/workflows/tidy-rust.yml@main test: strategy: @@ -35,9 +35,10 @@ jobs: # Note: Specifying the version of valgrind and cargo-binstall is not supported. - os: ubuntu-20.04 tool: cargo-hack@0.5.24,cargo-llvm-cov@0.5.3,cargo-minimal-versions@0.1.8,parse-changelog@0.5.2,cargo-udeps@0.1.35,cargo-valgrind@2.1.0,cargo-deny@0.13.5,cross@0.2.4,nextest@0.9.11,protoc@3.21.12,shellcheck@0.9.0,shfmt@3.6.0,wasm-pack@0.10.3,wasmtime@4.0.0,mdbook@0.4.25,mdbook-linkcheck@0.7.7,cargo-watch@8.1.1 - # Nextest supports basic version ranges as well. For other tools, this will be supported by https://github.com/taiki-e/install-action/pull/27. - os: ubuntu-20.04 - tool: nextest@0.9 + tool: cargo-hack@0.5,cargo-llvm-cov@0.5,cargo-minimal-versions@0.1,parse-changelog@0.5,cargo-udeps@0.1,cargo-valgrind@2.1,cargo-deny@0.13,cross@0.2,nextest@0.9,protoc@3.21,shellcheck@0.9,shfmt@3.5,wasm-pack@0.10,wasmtime@4.0,mdbook@0.4,mdbook-linkcheck@0.7,cargo-watch@8.1 + - os: ubuntu-20.04 + tool: cargo-valgrind@2,protoc@3,shfmt@3,wasmtime@4,cargo-watch@8 - os: macos-11 tool: cargo-hack,cargo-llvm-cov,cargo-minimal-versions,parse-changelog,cargo-udeps,cargo-valgrind,cargo-deny,cross,nextest,protoc,shellcheck,shfmt,wasm-pack,wasmtime,mdbook,mdbook-linkcheck,cargo-watch - os: windows-2019 @@ -47,6 +48,8 @@ jobs: - uses: actions/checkout@v3 with: persist-credentials: false + # cross attempts to install rust-src when Cargo.toml is available even if `cross --version` + - run: rm Cargo.toml - uses: ./ with: tool: ${{ matrix.tool }} @@ -113,6 +116,36 @@ jobs: - uses: actions/checkout@v3 with: persist-credentials: false + # cross attempts to install rust-src when Cargo.toml is available even if `cross --version` + - run: rm Cargo.toml - uses: ./ with: tool: ${{ matrix.tool }} + + manifest: + runs-on: ubuntu-latest + permissions: + contents: write # TODO test + pull-requests: write + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@nightly + - run: tools/manifest.sh + - run: git add -N . && git diff --exit-code + if: github.repository_owner != 'taiki-e' || github.event_name != 'schedule' && !(github.event_name == 'push' && github.ref == 'refs/heads/main') + - id: diff + run: ci/manifest.sh + if: github.repository_owner == 'taiki-e' && (github.event_name == 'schedule' || github.event_name == 'push' && github.ref == 'refs/heads/main') + - uses: taiki-e/create-pull-request@v4 + with: + title: Update manifest + body: | + Auto-generated by [create-pull-request][1] + [Please close and immediately reopen this pull request to run CI.][2] + + [1]: https://github.com/peter-evans/create-pull-request + [2]: https://github.com/peter-evans/create-pull-request/blob/HEAD/docs/concepts-guidelines.md#workarounds-to-trigger-further-workflow-runs + branch: update-manifest + if: github.repository_owner == 'taiki-e' && (github.event_name == 'schedule' || github.event_name == 'push' && github.ref == 'refs/heads/main') && steps.diff.outputs.success == 'false' diff --git a/.gitignore b/.gitignore index a5a03e45..2c324bff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ target Cargo.lock +tmp # For platform and editor specific settings, it is recommended to add to # a global .gitignore file. diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 00000000..3a26366d --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1 @@ +edition = "2021" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..3467e5fd --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,3 @@ +[workspace] +resolver = "2" +members = ["tools/codegen"] diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 00000000..d1161325 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,19 @@ +# Development Guide + +## Add support for new tool + +1\. Add base manifest to [`tools/codegen/base`](tools/codegen/base) directory. + +See JSON files in `tools/codegen/base` directory for examples of the manifest. + +2\. Generate manifest with the following command (replace `` with the tool name). + +```sh +./tools/manifest.sh +``` + +3\. Add tool name to table in "Supported tools" section in `README.md`. + +4\. Add tool name to `tools` variable in `tools/publish.sh`. + +5\. Add tool name to test matrix in `.github/workflows/ci.yml`. diff --git a/README.md b/README.md index dc814971..52051f57 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,10 @@ GitHub Action for installing development tools (mainly from GitHub Releases). ### Inputs -| Name | Required | Description | Type | Default | -| ---- |:--------:| ----------- | ---- | ------- | -| tool | **true** | Tools to install (comma-separated list) | String | | +| Name | Required | Description | Type | Default | +| -------- |:--------:| --------------------------------------- | ------- | ------- | +| tool | **true** | Tools to install (comma-separated list) | String | | +| checksum | false | Whether to enable checksums | Boolean | `true` | ### Example workflow @@ -100,7 +101,9 @@ If a tool not included in the list above is specified, this action uses [cargo-b When installing the tool from GitHub Releases, this action will download the tool or its installer from GitHub Releases using HTTPS with tlsv1.2+. This is basically considered to be the same level of security as [the recommended installation of rustup](https://www.rust-lang.org/tools/install). -If you want a higher level of security, consider working on [#1](https://github.com/taiki-e/install-action/issues/1). +Additionally, this action will also verify SHA256 checksums for downloaded files in all tools installed from GitHub Releases. This is enabled by default and can be disabled by setting the `checksum` input option to `false`. + +See the linked documentation for information on security when installed using [snap](https://snapcraft.io/docs) or [cargo-binstall](https://github.com/cargo-bins/cargo-binstall#faq). ## Compatibility diff --git a/action.yml b/action.yml index 0eac1f13..bee4b90d 100644 --- a/action.yml +++ b/action.yml @@ -6,6 +6,10 @@ inputs: description: Tools to install (comma-separated list) required: true # default: #publish:tool + checksum: + description: Whether to enable checksums + required: false + default: 'true' runs: using: node16 diff --git a/ci/manifest.sh b/ci/manifest.sh new file mode 100755 index 00000000..32233557 --- /dev/null +++ b/ci/manifest.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euxo pipefail +IFS=$'\n\t' +cd "$(dirname "$0")"/.. + +git config user.name "Taiki Endo" +git config user.email "te316e89@gmail.com" + +for manifest in manifests/*.json; do + git add -N "${manifest}" + if ! git diff --exit-code -- "${manifest}"; then + name="$(basename "${manifest%.*}")" + git add "${manifest}" + git commit -m "Update ${name}" + has_update=1 + fi +done + +if [[ -n "${has_update:-}" ]]; then + echo "success=false" >>"${GITHUB_OUTPUT}" +fi diff --git a/main.sh b/main.sh index 43331b16..c3591c3d 100755 --- a/main.sh +++ b/main.sh @@ -30,10 +30,33 @@ warn() { info() { echo "info: $*" } -download() { +download_and_checksum() { local url="$1" - local bin_dir="$2" - local bin="$3" + local checksum="$2" + if [[ -z "${enable_checksum}" ]]; then + checksum="" + fi + info "downloading ${url}" + retry curl --proto '=https' --tlsv1.2 -fsSL --retry 10 "${url}" -o tmp + if [[ -n "${checksum}" ]]; then + info "verifying sha256 checksum for $(basename "${url}")" + echo "${checksum} *tmp" >tmp.sha256sum + if type -P sha256sum &>/dev/null; then + sha256sum -c tmp.sha256sum >/dev/null + elif type -P shasum &>/dev/null; then + # GitHub-hosted macOS runner does not install GNU Coreutils by default. + # https://github.com/actions/runner-images/issues/90 + shasum -a 256 -c tmp.sha256sum >/dev/null + else + bail "checksum requires 'sha256sum' or 'shasum' command; consider installing one of them or setting 'checksum' input option to 'false'" + fi + fi +} +download_and_extract() { + local url="$1" + local checksum="$2" + local bin_dir="$3" + local bin_in_archive="$4" # path to bin in archive if [[ "${bin_dir}" == "/usr/"* ]]; then if [[ ! -d "${bin_dir}" ]]; then bin_dir="${HOME}/.install-action/bin" @@ -44,6 +67,9 @@ download() { fi fi fi + local installed_bin + installed_bin="${bin_dir}/$(basename "${bin_in_archive}")" + local tar_args=() case "${url}" in *.tar.gz | *.tgz) tar_args+=("xzf") ;; @@ -70,32 +96,100 @@ download() { debian | alpine | fedora) sys_install unzip ;; esac fi - mkdir -p .install-action-tmp - ( - cd .install-action-tmp - info "downloading ${url}" - retry curl --proto '=https' --tlsv1.2 -fsSL --retry 10 "${url}" -o tmp.zip - unzip tmp.zip - mv "${bin}" "${bin_dir}/" - ) - rm -rf .install-action-tmp - return 0 ;; - *) bail "unrecognized archive format '${url}' for ${tool}" ;; esac - tar_args+=("-") - local components - components=$(tr <<<"${bin}" -cd '/' | wc -c) - if [[ "${components}" != "0" ]]; then - tar_args+=(--strip-components "${components}") + + mkdir -p "${tmp_dir}" + ( + cd "${tmp_dir}" + download_and_checksum "${url}" "${checksum}" + if [[ ${#tar_args[@]} -gt 0 ]]; then + tar_args+=("tmp") + local components + components=$(tr <<<"${bin_in_archive}" -cd '/' | wc -c) + if [[ "${components}" != "0" ]]; then + tar_args+=(--strip-components "${components}") + fi + tar "${tar_args[@]}" -C "${bin_dir}" "${bin_in_archive}" + else + case "${url}" in + *.zip) + unzip tmp + mv "${bin_in_archive}" "${bin_dir}/" + ;; + *) mv tmp "${installed_bin}" ;; + esac + fi + ) + rm -rf "${tmp_dir}" + + case "${host_os}" in + linux | macos) + if [[ ! -x "${installed_bin}" ]]; then + chmod +x "${installed_bin}" + fi + ;; + esac +} +read_manifest() { + local tool="$1" + local version="$2" + local manifest + manifest=$(jq -r ".\"${version}\"" "${manifest_dir}/${tool}.json") + local download_info + case "${host_os}" in + linux) + # Static-linked binaries compiled for linux-musl will also work on linux-gnu systems and are + # usually preferred over linux-gnu binaries because they can avoid glibc version issues. + # (rustc enables statically linking for linux-musl by default, except for mips.) + download_info=$(jq <<<"${manifest}" -r ".${host_arch}_linux_musl") + if [[ "${download_info}" == "null" ]]; then + # Even if host_env is musl, we won't issue an error here because it seems that in + # some cases linux-gnu binaries will work on linux-musl hosts. + # https://wiki.alpinelinux.org/wiki/Running_glibc_programs + # TODO: However, a warning may make sense. + download_info=$(jq <<<"${manifest}" -r ".${host_arch}_linux_gnu") + elif [[ "${host_env}" == "gnu" ]]; then + case "${tool}" in + cargo-nextest | nextest) + # musl build of nextest is slow, so use glibc build if host_env is gnu. + # https://github.com/taiki-e/install-action/issues/13 + download_info=$(jq <<<"${manifest}" -r ".${host_arch}_linux_gnu") + ;; + esac + fi + ;; + macos | windows) + # Binaries compiled for x86_64 macOS will usually also work on aarch64 macOS. + # Binaries compiled for x86_64 Windows will usually also work on aarch64 Windows 11+. + download_info=$(jq <<<"${manifest}" -r ".${host_arch}_${host_os}") + if [[ "${download_info}" == "null" ]] && [[ "${host_arch}" != "x86_64" ]]; then + download_info=$(jq <<<"${manifest}" -r ".download_info.x86_64_${host_os}") + fi + ;; + *) bail "unsupported OS type '${host_os}' for ${tool}" ;; + esac + if [[ "${download_info}" == "null" ]]; then + bail "${tool}@${version} for '${host_os}' is not supported" fi - info "downloading ${url}" - retry curl --proto '=https' --tlsv1.2 -fsSL --retry 10 "${url}" \ - | tar "${tar_args[@]}" -C "${bin_dir}" "${bin}" + url=$(jq <<<"${download_info}" -r '.url') + checksum=$(jq <<<"${download_info}" -r '.checksum') + bin_dir=$(jq <<<"${download_info}" -r '.bin_dir') + bin_in_archive=$(jq <<<"${download_info}" -r '.bin') + if [[ "${bin_dir}" == "null" ]]; then + bin_dir="${cargo_bin}" + fi + if [[ "${bin_in_archive}" == "null" ]]; then + bin_in_archive="${tool}${exe}" + fi +} +download_from_manifest() { + read_manifest "$@" + download_and_extract "${url}" "${checksum}" "${bin_dir}" "${bin_in_archive}" } install_cargo_binstall() { - # https://github.com/cargo-bins/cargo-binstall/releases - local binstall_version="0.18.1" + local binstall_version + binstall_version=$(jq -r '.latest.version' "${manifest_dir}/cargo-binstall.json") local install_binstall='1' if [[ -f "${cargo_bin}/cargo-binstall${exe}" ]]; then if [[ "$(cargo binstall -V)" == "cargo-binstall ${binstall_version}" ]]; then @@ -109,16 +203,7 @@ install_cargo_binstall() { if [[ -n "${install_binstall}" ]]; then info "installing cargo-binstall" - - base_url="https://github.com/cargo-bins/cargo-binstall/releases/download/v${binstall_version}/cargo-binstall" - case "${OSTYPE}" in - linux*) url="${base_url}-${host_arch}-unknown-linux-musl.tgz" ;; - darwin*) url="${base_url}-${host_arch}-apple-darwin.zip" ;; - cygwin* | msys*) url="${base_url}-x86_64-pc-windows-msvc.zip" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for cargo-binstall" ;; - esac - - download "${url}" "${cargo_bin}" "cargo-binstall${exe}" + download_from_manifest "cargo-binstall" "latest" info "cargo-binstall installed at $(type -P "cargo-binstall${exe}")" x cargo binstall -V fi @@ -198,6 +283,7 @@ if [[ $# -gt 0 ]]; then fi export DEBIAN_FRONTEND=noninteractive +manifest_dir="$(dirname "$0")/manifests" # Inputs tool="${INPUT_TOOL:-}" @@ -206,6 +292,13 @@ if [[ -n "${tool}" ]]; then while read -rd,; do tools+=("${REPLY}"); done <<<"${tool}," fi +enable_checksum="${INPUT_CHECKSUM:-}" +case "${enable_checksum}" in + true) ;; + false) enable_checksum='' ;; + *) bail "'checksum' input option must be 'true' or 'false': '${enable_checksum}'" ;; +esac + # Refs: https://github.com/rust-lang/rustup/blob/HEAD/rustup-init.sh case "$(uname -m)" in aarch64 | arm64) host_arch="aarch64" ;; @@ -226,8 +319,9 @@ case "$(uname -m)" in esac base_distro="" exe="" -case "${OSTYPE}" in - linux*) +case "$(uname -s)" in + Linux) + host_os=linux host_env="gnu" if (ldd --version 2>&1 || true) | grep -q 'musl'; then host_env="musl" @@ -260,17 +354,32 @@ case "${OSTYPE}" in ;; esac ;; - cygwin* | msys*) exe=".exe" ;; + Darwin) host_os=macos ;; + MINGW* | MSYS* | CYGWIN* | Windows_NT) + host_os=windows + exe=".exe" + ;; + *) bail "unrecognized OS type '$(uname -s)'" ;; esac +tmp_dir="${HOME}/.install-action/tmp" cargo_bin="${CARGO_HOME:-"${HOME}/.cargo"}/bin" if [[ ! -d "${cargo_bin}" ]]; then cargo_bin=/usr/local/bin fi -if ! type -P curl &>/dev/null || ! type -P tar &>/dev/null; then +if ! type -P jq &>/dev/null || ! type -P curl &>/dev/null || ! type -P tar &>/dev/null; then case "${base_distro}" in - debian | alpine | fedora) sys_install ca-certificates curl tar ;; + debian | alpine) sys_install ca-certificates curl jq tar ;; + fedora) + if [[ "${dnf}" == "yum" ]]; then + # On RHEL7-based distribution jq requires EPEL + sys_install ca-certificates curl tar epel-release + sys_install jq --enablerepo=epel + else + sys_install ca-certificates curl jq tar + fi + ;; esac fi @@ -284,154 +393,10 @@ for tool in "${tools[@]}"; do version="latest" fi tool="${tool%@*}" - bin="${tool}${exe}" info "installing ${tool}@${version}" case "${tool}" in - cargo-hack | cargo-llvm-cov | cargo-minimal-versions | parse-changelog) - case "${tool}" in - # https://github.com/taiki-e/cargo-hack/releases - cargo-hack) latest_version="0.5.24" ;; - # https://github.com/taiki-e/cargo-llvm-cov/releases - cargo-llvm-cov) latest_version="0.5.3" ;; - # https://github.com/taiki-e/cargo-minimal-versions/releases - cargo-minimal-versions) latest_version="0.1.8" ;; - # https://github.com/taiki-e/parse-changelog/releases - parse-changelog) latest_version="0.5.2" ;; - *) exit 1 ;; - esac - repo="taiki-e/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - case "${OSTYPE}" in - linux*) target="${host_arch}-unknown-linux-musl" ;; - darwin*) target="${host_arch}-apple-darwin" ;; - cygwin* | msys*) - case "${tool}" in - cargo-llvm-cov) target="x86_64-pc-windows-msvc" ;; - *) target="${host_arch}-pc-windows-msvc" ;; - esac - ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - url="https://github.com/${repo}/releases/download/v${version}/${tool}-${target}.tar.gz" - download "${url}" "${cargo_bin}" "${tool}${exe}" - ;; - cargo-udeps) - # https://github.com/est31/cargo-udeps/releases - latest_version="0.1.35" - repo="est31/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - base_url="https://github.com/${repo}/releases/download/v${version}/${tool}-v${version}" - case "${OSTYPE}" in - linux*) - target="x86_64-unknown-linux-gnu" - url="${base_url}-${target}.tar.gz" - ;; - darwin*) - target="x86_64-apple-darwin" - url="${base_url}-${target}.tar.gz" - ;; - cygwin* | msys*) - target="x86_64-pc-windows-msvc" - url="${base_url}-${target}.zip" - ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - # leading `./` is required for cargo-udeps to work - download "${url}" "${cargo_bin}" "./${tool}-v${version}-${target}/${tool}${exe}" - ;; - cargo-valgrind) - # https://github.com/jfrimmel/cargo-valgrind/releases - latest_version="2.1.0" - repo="jfrimmel/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - base_url="https://github.com/${repo}/releases/download/v${version}/${tool}-${version}" - case "${OSTYPE}" in - linux*) - target="x86_64-unknown-linux-musl" - url="${base_url}-${target}.tar.gz" - ;; - darwin*) - target="x86_64-apple-darwin" - url="${base_url}-${target}.tar.gz" - ;; - cygwin* | msys*) - target="x86_64-pc-windows-msvc" - url="${base_url}-${target}.zip" - ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - download "${url}" "${cargo_bin}" "${tool}${exe}" - ;; - cargo-deny) - # https://github.com/EmbarkStudios/cargo-deny/releases - latest_version="0.13.5" - repo="EmbarkStudios/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - case "${OSTYPE}" in - linux*) target="x86_64-unknown-linux-musl" ;; - darwin*) target="${host_arch}-apple-darwin" ;; - cygwin* | msys*) target="x86_64-pc-windows-msvc" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - url="https://github.com/${repo}/releases/download/${version}/${tool}-${version}-${target}.tar.gz" - download "${url}" "${cargo_bin}" "${tool}-${version}-${target}/${tool}${exe}" - ;; - cross) - # https://github.com/cross-rs/cross/releases - latest_version="0.2.4" - repo="cross-rs/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - case "${OSTYPE}" in - linux*) target="x86_64-unknown-linux-musl" ;; - darwin*) target="x86_64-apple-darwin" ;; - cygwin* | msys*) target="x86_64-pc-windows-msvc" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - case "${version}" in - 0.1.* | 0.2.[0-1]) url="https://github.com/${repo}/releases/download/v${version}/${tool}-v${version}-${target}.tar.gz" ;; - *) url="https://github.com/${repo}/releases/download/v${version}/${tool}-${target}.tar.gz" ;; - esac - download "${url}" "${cargo_bin}" "${tool}${exe}" - ;; - nextest | cargo-nextest) - bin="cargo-nextest" - # https://nexte.st/book/pre-built-binaries.html - case "${OSTYPE}" in - linux*) - # musl build of nextest is slow, so use glibc build if host_env is gnu. - # https://github.com/taiki-e/install-action/issues/13 - case "${host_env}" in - gnu) url="https://get.nexte.st/${version}/linux" ;; - *) url="https://get.nexte.st/${version}/linux-musl" ;; - esac - ;; - darwin*) url="https://get.nexte.st/${version}/mac" ;; - cygwin* | msys*) url="https://get.nexte.st/${version}/windows-tar" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - info "downloading ${url}" - retry curl --proto '=https' --tlsv1.2 -fsSL --retry 10 "${url}" \ - | tar xzf - -C "${cargo_bin}" - ;; protoc) - # https://github.com/protocolbuffers/protobuf/releases - latest_version="3.21.12" - repo="protocolbuffers/protobuf" - case "${version}" in - latest) version="${latest_version}" ;; - esac - miner_patch_version="${version#*.}" - base_url="https://github.com/${repo}/releases/download/v${miner_patch_version}/protoc-${miner_patch_version}" + read_manifest "protoc" "${version}" # Copying files to /usr/local/include requires sudo, so do not use it. bin_dir="${HOME}/.install-action/bin" include_dir="${HOME}/.install-action/include" @@ -441,109 +406,38 @@ for tool in "${tools[@]}"; do echo "${bin_dir}" >>"${GITHUB_PATH}" export PATH="${PATH}:${bin_dir}" fi - case "${OSTYPE}" in - linux*) url="${base_url}-linux-${host_arch/aarch/aarch_}.zip" ;; - darwin*) url="${base_url}-osx-${host_arch/aarch/aarch_}.zip" ;; - cygwin* | msys*) url="${base_url}-win64.zip" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac if ! type -P unzip &>/dev/null; then case "${base_distro}" in debian | alpine | fedora) sys_install unzip ;; esac fi - mkdir -p .install-action-tmp + mkdir -p "${tmp_dir}" ( - cd .install-action-tmp - info "downloading ${url}" - retry curl --proto '=https' --tlsv1.2 -fsSL --retry 10 "${url}" -o tmp.zip - unzip tmp.zip + cd "${tmp_dir}" + download_and_checksum "${url}" "${checksum}" + unzip tmp mv "bin/protoc${exe}" "${bin_dir}/" mkdir -p "${include_dir}/" cp -r include/. "${include_dir}/" - case "${OSTYPE}" in - cygwin* | msys*) bin_dir=$(sed <<<"${bin_dir}" 's/^\/c\//C:\\/') ;; + case "${host_os}" in + windows) bin_dir=$(sed <<<"${bin_dir}" 's/^\/c\//C:\\/') ;; esac if [[ -z "${PROTOC:-}" ]]; then info "setting PROTOC environment variable" echo "PROTOC=${bin_dir}/protoc${exe}" >>"${GITHUB_ENV}" fi ) - rm -rf .install-action-tmp - ;; - shellcheck) - # https://github.com/koalaman/shellcheck/releases - latest_version="0.9.0" - repo="koalaman/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - base_url="https://github.com/${repo}/releases/download/v${version}/${tool}-v${version}" - bin="${tool}-v${version}/${tool}${exe}" - case "${OSTYPE}" in - linux*) - if type -P shellcheck &>/dev/null; then - apt_remove shellcheck - fi - url="${base_url}.linux.${host_arch}.tar.xz" - ;; - darwin*) url="${base_url}.darwin.x86_64.tar.xz" ;; - cygwin* | msys*) - url="${base_url}.zip" - bin="${tool}${exe}" - ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - download "${url}" /usr/local/bin "${bin}" - ;; - shfmt) - # https://github.com/mvdan/sh/releases - latest_version="3.6.0" - repo="mvdan/sh" - case "${version}" in - latest) version="${latest_version}" ;; - esac - bin_dir="/usr/local/bin" - case "${OSTYPE}" in - linux*) - case "${host_arch}" in - aarch64) target="linux_arm64" ;; - *) target="linux_amd64" ;; - esac - ;; - darwin*) - case "${host_arch}" in - aarch64) target="darwin_arm64" ;; - *) target="darwin_amd64" ;; - esac - ;; - cygwin* | msys*) - target="windows_amd64" - bin_dir="${HOME}/.install-action/bin" - if [[ ! -d "${bin_dir}" ]]; then - mkdir -p "${bin_dir}" - echo "${bin_dir}" >>"${GITHUB_PATH}" - export PATH="${PATH}:${bin_dir}" - fi - ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - url="https://github.com/${repo}/releases/download/v${version}/${tool}_v${version}_${target}${exe}" - info "downloading ${url}" - retry curl --proto '=https' --tlsv1.2 -fsSL --retry 10 -o "${bin_dir}/${tool}${exe}" "${url}" - case "${OSTYPE}" in - linux* | darwin*) chmod +x "${bin_dir}/${tool}${exe}" ;; - esac + rm -rf "${tmp_dir}" ;; valgrind) case "${version}" in latest) ;; *) warn "specifying the version of ${tool} is not supported yet by this action" ;; esac - case "${OSTYPE}" in - linux*) ;; - darwin* | cygwin* | msys*) bail "${tool} for non-linux is not supported yet by this action" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; + case "${host_os}" in + linux) ;; + macos | windows) bail "${tool} for non-linux is not supported yet by this action" ;; + *) bail "unsupported host OS '${host_os}' for ${tool}" ;; esac # libc6-dbg is needed to run Valgrind apt_install libc6-dbg @@ -551,102 +445,48 @@ for tool in "${tools[@]}"; do # https://snapcraft.io/install/valgrind/ubuntu snap_install valgrind --classic ;; - wasm-pack) - # https://github.com/rustwasm/wasm-pack/releases - latest_version="0.10.3" - repo="rustwasm/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - case "${OSTYPE}" in - linux*) target="${host_arch}-unknown-linux-musl" ;; - darwin*) target="x86_64-apple-darwin" ;; - cygwin* | msys*) target="x86_64-pc-windows-msvc" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - url="https://github.com/${repo}/releases/download/v${version}/${tool}-v${version}-${target}.tar.gz" - download "${url}" "${cargo_bin}" "${tool}-v${version}-${target}/${tool}${exe}" - ;; - wasmtime) - # https://github.com/bytecodealliance/wasmtime/releases - latest_version="4.0.0" - repo="bytecodealliance/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - base_url="https://github.com/${repo}/releases/download/v${version}/${tool}-v${version}" - case "${OSTYPE}" in - linux*) - target="${host_arch}-linux" - url="${base_url}-${target}.tar.xz" - ;; - darwin*) - target="${host_arch}-macos" - url="${base_url}-${target}.tar.xz" - ;; - cygwin* | msys*) - target="x86_64-windows" - url="${base_url}-${target}.zip" - ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - download "${url}" "${cargo_bin}" "${tool}-v${version}-${target}/${tool}${exe}" - ;; - mdbook) - # https://github.com/rust-lang/mdBook/releases - latest_version="0.4.25" - repo="rust-lang/mdBook" - case "${version}" in - latest) version="${latest_version}" ;; - esac - base_url="https://github.com/${repo}/releases/download/v${version}/${tool}-v${version}" - case "${OSTYPE}" in - linux*) - case "${version}" in - 0.[1-3].* | 0.4.? | 0.4.1? | 0.4.2[0-1]) url="${base_url}-x86_64-unknown-linux-gnu.tar.gz" ;; - *) url="${base_url}-${host_arch}-unknown-linux-musl.tar.gz" ;; - esac - ;; - darwin*) url="${base_url}-x86_64-apple-darwin.tar.gz" ;; - cygwin* | msys*) url="${base_url}-x86_64-pc-windows-msvc.zip" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - download "${url}" "${cargo_bin}" "${tool}${exe}" - ;; - mdbook-linkcheck) - # https://github.com/Michael-F-Bryan/mdbook-linkcheck/releases - latest_version="0.7.7" - repo="Michael-F-Bryan/${tool}" - case "${version}" in - latest) version="${latest_version}" ;; - esac - case "${OSTYPE}" in - linux*) target="x86_64-unknown-linux-gnu" ;; - darwin*) target="x86_64-apple-darwin" ;; - cygwin* | msys*) target="x86_64-pc-windows-msvc" ;; - *) bail "unsupported OSTYPE '${OSTYPE}' for ${tool}" ;; - esac - url="https://github.com/${repo}/releases/download/v${version}/${tool}.${target}.zip" - download "${url}" "${cargo_bin}" "${tool}${exe}" - case "${OSTYPE}" in - linux* | darwin*) chmod +x "${cargo_bin}/${tool}${exe}" ;; - esac - ;; cargo-binstall) + case "${version}" in + latest) ;; + *) warn "specifying the version of ${tool} is not supported by this action" ;; + esac install_cargo_binstall echo continue ;; *) - cargo_binstall "${tool}" "${version}" - continue + # Handle aliases + case "${tool}" in + cargo-nextest | nextest) tool="cargo-nextest" ;; + esac + + # Use cargo-binstall fallback if tool is not available. + if [[ ! -f "${manifest_dir}/${tool}.json" ]]; then + cargo_binstall "${tool}" "${version}" + continue + fi + + # Pre-install + case "${tool}" in + shellcheck) + case "${host_os}" in + linux) + if type -P shellcheck &>/dev/null; then + apt_remove -y shellcheck + fi + ;; + esac + ;; + esac + + download_from_manifest "${tool}" "${version}" ;; esac - info "${tool} installed at $(type -P "${bin}")" - case "${bin}" in - "cargo-udeps${exe}") x cargo udeps --help | head -1 ;; # cargo-udeps v0.1.30 does not support --version option - "cargo-valgrind${exe}") x cargo valgrind --help ;; # cargo-valgrind v2.1.0 does not support --version option + info "${tool} installed at $(type -P "${tool}${exe}")" + case "${tool}" in + cargo-udeps) x cargo udeps --help | head -1 ;; # cargo-udeps v0.1.30 does not support --version option + cargo-valgrind) x cargo valgrind --help ;; # cargo-valgrind v2.1.0 does not support --version option cargo-*) x cargo "${tool#cargo-}" --version ;; *) x "${tool}" --version ;; esac diff --git a/tools/codegen/Cargo.toml b/tools/codegen/Cargo.toml new file mode 100644 index 00000000..06aab29c --- /dev/null +++ b/tools/codegen/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "install-action-internal-codegen" +version = "0.0.0" +edition = "2021" +publish = false + +[dependencies] +anyhow = "1" +fs-err = "2" +semver = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +ureq = { version = "2", features = ["json"] } diff --git a/tools/codegen/base/cargo-binstall.json b/tools/codegen/base/cargo-binstall.json new file mode 100644 index 00000000..a06112c7 --- /dev/null +++ b/tools/codegen/base/cargo-binstall.json @@ -0,0 +1,19 @@ +{ + "repository": "https://github.com/cargo-bins/cargo-binstall", + "tag_prefix": "v", + "asset_name": "${package}-${rust_target}.tgz", + "version_range": "latest", + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": { + "asset_name": "${package}-${rust_target}.zip" + }, + "x86_64_windows": { + "asset_name": "${package}-${rust_target}.zip" + }, + "aarch64_linux_musl": {}, + "aarch64_macos": { + "asset_name": "${package}-${rust_target}.zip" + } + } +} diff --git a/tools/codegen/base/cargo-deny.json b/tools/codegen/base/cargo-deny.json new file mode 100644 index 00000000..c994a904 --- /dev/null +++ b/tools/codegen/base/cargo-deny.json @@ -0,0 +1,12 @@ +{ + "repository": "https://github.com/EmbarkStudios/cargo-deny", + "tag_prefix": "", + "asset_name": "${package}-${version}-${rust_target}.tar.gz", + "bin": "${package}-${version}-${rust_target}/${package}${exe}", + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": {}, + "aarch64_macos": {} + } +} diff --git a/tools/codegen/base/cargo-hack.json b/tools/codegen/base/cargo-hack.json new file mode 100644 index 00000000..d28801be --- /dev/null +++ b/tools/codegen/base/cargo-hack.json @@ -0,0 +1,22 @@ +{ + "repository": "https://github.com/taiki-e/cargo-hack", + "tag_prefix": "v", + "asset_name": [ + "${package}-${rust_target}.tar.gz", + "${package}-v${version}-${rust_target}.tar.gz" + ], + "platform": { + "x86_64_linux_gnu": {}, + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": { + "asset_name": [ + "${package}-${rust_target}.zip", + "${package}-v${version}-${rust_target}.zip" + ] + }, + "aarch64_linux_musl": {}, + "aarch64_macos": {}, + "aarch64_windows": {} + } +} diff --git a/tools/codegen/base/cargo-llvm-cov.json b/tools/codegen/base/cargo-llvm-cov.json new file mode 100644 index 00000000..024f5004 --- /dev/null +++ b/tools/codegen/base/cargo-llvm-cov.json @@ -0,0 +1,12 @@ +{ + "repository": "https://github.com/taiki-e/cargo-llvm-cov", + "tag_prefix": "v", + "asset_name": "${package}-${rust_target}.tar.gz", + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": {}, + "aarch64_linux_musl": {}, + "aarch64_macos": {} + } +} diff --git a/tools/codegen/base/cargo-minimal-versions.json b/tools/codegen/base/cargo-minimal-versions.json new file mode 100644 index 00000000..25789904 --- /dev/null +++ b/tools/codegen/base/cargo-minimal-versions.json @@ -0,0 +1,13 @@ +{ + "repository": "https://github.com/taiki-e/cargo-minimal-versions", + "tag_prefix": "v", + "asset_name": "${package}-${rust_target}.tar.gz", + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": {}, + "aarch64_linux_musl": {}, + "aarch64_macos": {}, + "aarch64_windows": {} + } +} diff --git a/tools/codegen/base/cargo-nextest.json b/tools/codegen/base/cargo-nextest.json new file mode 100644 index 00000000..41ad233b --- /dev/null +++ b/tools/codegen/base/cargo-nextest.json @@ -0,0 +1,15 @@ +{ + "repository": "https://github.com/nextest-rs/nextest", + "tag_prefix": "cargo-nextest-", + "asset_name": "${package}-${version}-${rust_target}.tar.gz", + "platform": { + "x86_64_linux_gnu": {}, + "x86_64_linux_musl": {}, + "x86_64_macos": { + "asset_name": "${package}-${version}-universal-apple-darwin.tar.gz" + }, + "x86_64_windows": {}, + "aarch64_linux_gnu": {} + }, + "prefer_linux_gnu": true +} diff --git a/tools/codegen/base/cargo-udeps.json b/tools/codegen/base/cargo-udeps.json new file mode 100644 index 00000000..a326e915 --- /dev/null +++ b/tools/codegen/base/cargo-udeps.json @@ -0,0 +1,13 @@ +{ + "repository": "https://github.com/est31/cargo-udeps", + "tag_prefix": "v", + "asset_name": "${package}-v${version}-${rust_target}.tar.gz", + "bin": "./${package}-v${version}-${rust_target}/${package}${exe}", + "platform": { + "x86_64_linux_gnu": {}, + "x86_64_macos": {}, + "x86_64_windows": { + "asset_name": "${package}-v${version}-${rust_target}.zip" + } + } +} diff --git a/tools/codegen/base/cargo-valgrind.json b/tools/codegen/base/cargo-valgrind.json new file mode 100644 index 00000000..b3332c3b --- /dev/null +++ b/tools/codegen/base/cargo-valgrind.json @@ -0,0 +1,12 @@ +{ + "repository": "https://github.com/jfrimmel/cargo-valgrind", + "tag_prefix": "v", + "asset_name": "${package}-${version}-${rust_target}.tar.gz", + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": { + "asset_name": "${package}-${version}-${rust_target}.zip" + } + } +} diff --git a/tools/codegen/base/cross.json b/tools/codegen/base/cross.json new file mode 100644 index 00000000..25438009 --- /dev/null +++ b/tools/codegen/base/cross.json @@ -0,0 +1,13 @@ +{ + "repository": "https://github.com/cross-rs/cross", + "tag_prefix": "v", + "asset_name": [ + "${package}-${rust_target}.tar.gz", + "${package}-v${version}-${rust_target}.tar.gz" + ], + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": {} + } +} diff --git a/tools/codegen/base/mdbook-linkcheck.json b/tools/codegen/base/mdbook-linkcheck.json new file mode 100644 index 00000000..e25c86af --- /dev/null +++ b/tools/codegen/base/mdbook-linkcheck.json @@ -0,0 +1,10 @@ +{ + "repository": "https://github.com/Michael-F-Bryan/mdbook-linkcheck", + "tag_prefix": "v", + "asset_name": "${package}.${rust_target}.zip", + "platform": { + "x86_64_linux_gnu": {}, + "x86_64_macos": {}, + "x86_64_windows": {} + } +} diff --git a/tools/codegen/base/mdbook.json b/tools/codegen/base/mdbook.json new file mode 100644 index 00000000..755bc715 --- /dev/null +++ b/tools/codegen/base/mdbook.json @@ -0,0 +1,14 @@ +{ + "repository": "https://github.com/rust-lang/mdBook", + "tag_prefix": "v", + "asset_name": "${package}-v${version}-${rust_target}.tar.gz", + "platform": { + "x86_64_linux_musl": {}, + "x86_64_linux_gnu": {}, + "x86_64_macos": {}, + "x86_64_windows": { + "asset_name": "${package}-v${version}-${rust_target}.zip" + }, + "aarch64_linux_musl": {} + } +} diff --git a/tools/codegen/base/parse-changelog.json b/tools/codegen/base/parse-changelog.json new file mode 100644 index 00000000..b1890cb8 --- /dev/null +++ b/tools/codegen/base/parse-changelog.json @@ -0,0 +1,16 @@ +{ + "repository": "https://github.com/taiki-e/parse-changelog", + "tag_prefix": "v", + "asset_name": "${package}-${rust_target}.tar.gz", + "platform": { + "x86_64_linux_gnu": {}, + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": { + "asset_name": "${package}-${rust_target}.zip" + }, + "aarch64_linux_musl": {}, + "aarch64_macos": {}, + "aarch64_windows": {} + } +} diff --git a/tools/codegen/base/protoc.json b/tools/codegen/base/protoc.json new file mode 100644 index 00000000..1e37650c --- /dev/null +++ b/tools/codegen/base/protoc.json @@ -0,0 +1,22 @@ +{ + "repository": "https://github.com/protocolbuffers/protobuf", + "tag_prefix": "v", + "default_major_version": "3", + "platform": { + "x86_64_linux_gnu": { + "asset_name": "${package}-${version}-linux-x86_64.zip" + }, + "x86_64_macos": { + "asset_name": "${package}-${version}-osx-x86_64.zip" + }, + "x86_64_windows": { + "asset_name": "${package}-${version}-win64.zip" + }, + "aarch64_linux_gnu": { + "asset_name": "${package}-${version}-linux-aarch_64.zip" + }, + "aarch64_macos": { + "asset_name": "${package}-${version}-osx-aarch_64.zip" + } + } +} diff --git a/tools/codegen/base/shellcheck.json b/tools/codegen/base/shellcheck.json new file mode 100644 index 00000000..0e68b7d0 --- /dev/null +++ b/tools/codegen/base/shellcheck.json @@ -0,0 +1,21 @@ +{ + "repository": "https://github.com/koalaman/shellcheck", + "tag_prefix": "v", + "bin_dir": "/usr/local/bin", + "bin": "${package}-v${version}/${package}${exe}", + "platform": { + "x86_64_linux_gnu": { + "asset_name": "${package}-v${version}.linux.x86_64.tar.xz" + }, + "x86_64_macos": { + "asset_name": "${package}-v${version}.darwin.x86_64.tar.xz" + }, + "x86_64_windows": { + "asset_name": "${package}-v${version}.zip", + "bin": "${package}${exe}" + }, + "aarch64_linux_gnu": { + "asset_name": "${package}-v${version}.linux.aarch64.tar.xz" + } + } +} diff --git a/tools/codegen/base/shfmt.json b/tools/codegen/base/shfmt.json new file mode 100644 index 00000000..c2c4dba8 --- /dev/null +++ b/tools/codegen/base/shfmt.json @@ -0,0 +1,22 @@ +{ + "repository": "https://github.com/mvdan/sh", + "tag_prefix": "v", + "bin_dir": "/usr/local/bin", + "platform": { + "x86_64_linux_gnu": { + "asset_name": "${package}_v${version}_linux_amd64" + }, + "x86_64_macos": { + "asset_name": "${package}_v${version}_darwin_amd64" + }, + "x86_64_windows": { + "asset_name": "${package}_v${version}_windows_amd64${exe}" + }, + "aarch64_linux_gnu": { + "asset_name": "${package}_v${version}_linux_arm64" + }, + "aarch64_macos": { + "asset_name": "${package}_v${version}_darwin_arm64" + } + } +} diff --git a/tools/codegen/base/wasm-pack.json b/tools/codegen/base/wasm-pack.json new file mode 100644 index 00000000..7056128b --- /dev/null +++ b/tools/codegen/base/wasm-pack.json @@ -0,0 +1,12 @@ +{ + "repository": "https://github.com/rustwasm/wasm-pack", + "tag_prefix": "v", + "asset_name": "${package}-v${version}-${rust_target}.tar.gz", + "bin": "${package}-v${version}-${rust_target}/${package}${exe}", + "platform": { + "x86_64_linux_musl": {}, + "x86_64_macos": {}, + "x86_64_windows": {}, + "aarch64_linux_musl": {} + } +} diff --git a/tools/codegen/base/wasmtime.json b/tools/codegen/base/wasmtime.json new file mode 100644 index 00000000..7a34c4ca --- /dev/null +++ b/tools/codegen/base/wasmtime.json @@ -0,0 +1,26 @@ +{ + "repository": "https://github.com/bytecodealliance/wasmtime", + "tag_prefix": "v", + "platform": { + "x86_64_linux_gnu": { + "asset_name": "${package}-v${version}-x86_64-linux.tar.xz", + "bin": "${package}-v${version}-x86_64-linux/${package}${exe}" + }, + "x86_64_macos": { + "asset_name": "${package}-v${version}-x86_64-macos.tar.xz", + "bin": "${package}-v${version}-x86_64-macos/${package}${exe}" + }, + "x86_64_windows": { + "asset_name": "${package}-v${version}-x86_64-windows.zip", + "bin": "${package}-v${version}-x86_64-windows/${package}${exe}" + }, + "aarch64_linux_gnu": { + "asset_name": "${package}-v${version}-aarch64-linux.tar.xz", + "bin": "${package}-v${version}-aarch64-linux/${package}${exe}" + }, + "aarch64_macos": { + "asset_name": "${package}-v${version}-aarch64-macos.tar.xz", + "bin": "${package}-v${version}-aarch64-macos/${package}${exe}" + } + } +} diff --git a/tools/codegen/src/main.rs b/tools/codegen/src/main.rs new file mode 100644 index 00000000..28fc7d31 --- /dev/null +++ b/tools/codegen/src/main.rs @@ -0,0 +1,549 @@ +use anyhow::{Context as _, Result}; +use fs_err as fs; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + cmp::Reverse, + collections::{BTreeMap, BTreeSet}, + env, fmt, + io::Read, + path::Path, + slice, + str::FromStr, +}; + +fn main() -> Result<()> { + let args: Vec<_> = env::args().skip(1).collect(); + if args.is_empty() || args.iter().any(|arg| arg.starts_with('-')) { + println!( + "USAGE: cargo run -p install-action-internal-codegen -r -- [VERSION_REQ]" + ); + std::process::exit(1); + } + let package = &args[0]; + + let root_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize()?; + let manifest_path = &root_dir.join("manifests").join(format!("{package}.json")); + let download_cache_dir = &root_dir.join("tools/codegen/tmp/cache").join(package); + fs::create_dir_all(download_cache_dir)?; + let base_info: BaseManifest = serde_json::from_slice(&fs::read( + root_dir + .join("tools/codegen/base") + .join(format!("{package}.json")), + )?)?; + let repo = base_info + .repository + .strip_prefix("https://github.com/") + .context("repository must be starts with https://github.com/")?; + eprintln!("downloading releases of https://github.com/{repo}"); + let mut releases: github::Releases = vec![]; + for page in 1.. { + let per_page = 100; + let mut req = ureq::get(&format!( + "https://api.github.com/repos/{repo}/releases?per_page={per_page}&page={page}" + )); + if let Ok(token) = env::var("INTERNAL_CODEGEN_GH_PAT") { + req = req.set("Authorization", &token); + } + let mut r: github::Releases = req.call()?.into_json()?; + if r.len() < per_page { + releases.append(&mut r); + break; + } + releases.append(&mut r); + } + let releases: Vec<_> = releases + .iter() + .filter_map(|release| { + release + .tag_name + .strip_prefix(&base_info.tag_prefix) + .map(|version| (version, release)) + }) + .collect(); + + let mut manifests: Manifests = BTreeMap::new(); + let mut semver_versions = BTreeSet::new(); + let mut has_build_metadata = false; + + let mut latest_only = false; + if let Some(version_range) = &base_info.version_range { + if version_range == "latest" { + latest_only = true; + } + } + let version_req: Option = match args.get(1) { + _ if latest_only => Some(format!("={}", releases.first().unwrap().0).parse()?), + None => match base_info.version_range { + Some(version_range) => Some(version_range.parse()?), + None => Some(">= 0.0.1".parse()?), // HACK: ignore pre-releases + }, + Some(version_req) => { + if manifest_path.is_file() { + match serde_json::from_slice(&fs::read(manifest_path)?) { + Ok(m) => manifests = m, + Err(e) => eprintln!("failed to load old manifest: {e}"), + } + } + for version in manifests.keys() { + let Some(semver_version) = version.0.to_semver() else { + continue; + }; + has_build_metadata |= !semver_version.build.is_empty(); + if semver_version.pre.is_empty() { + semver_versions.insert(semver_version.clone()); + } + } + + let req = if version_req == "latest" { + if manifests.is_empty() { + format!("={}", releases.first().unwrap().0).parse()? + } else { + format!(">={}", semver_versions.last().unwrap()).parse()? + } + } else { + version_req.parse()? + }; + eprintln!("update manifest for versions '{req}'"); + Some(req) + } + }; + + let mut buf = vec![]; + for &(version, release) in &releases { + let mut semver_version = version.parse::(); + if semver_version.is_err() { + if let Some(default_major_version) = &base_info.default_major_version { + semver_version = format!("{default_major_version}.{version}").parse(); + } + } + let Ok(semver_version) = semver_version else { + continue; + }; + if let Some(version_req) = &version_req { + if !version_req.matches(&semver_version) { + continue; + } + } + let mut download_info = BTreeMap::new(); + for (&platform, base_download_info) in &base_info.platform { + let asset_names = base_download_info + .asset_name + .as_ref() + .or(base_info.asset_name.as_ref()) + .with_context(|| format!("asset_name is needed for {package} on {platform:?}"))? + .as_slice() + .iter() + .map(|asset_name| replace_vars(asset_name, package, version, platform)) + .collect::>(); + let (url, asset_name) = match asset_names.iter().find_map(|asset_name| { + release + .assets + .iter() + .find(|asset| asset.name == *asset_name) + .map(|asset| (asset, asset_name)) + }) { + Some((asset, asset_name)) => { + (asset.browser_download_url.clone(), asset_name.clone()) + } + None => { + eprintln!("no asset '{asset_names:?}' for host platform '{platform:?}'"); + continue; + } + }; + + eprintln!("downloading {url} for checksum"); + let download_cache = download_cache_dir.join(format!( + "{version}-{platform:?}-{}", + Path::new(&url).file_name().unwrap().to_str().unwrap() + )); + if download_cache.is_file() { + eprintln!("{url} is already downloaded"); + fs::File::open(download_cache)?.read_to_end(&mut buf)?; + } else { + ureq::get(&url) + .call()? + .into_reader() + .read_to_end(&mut buf)?; + eprintln!("downloaded complete"); + fs::write(download_cache, &buf)?; + } + eprintln!("getting sha256 hash for {url}"); + let hash = Sha256::digest(&buf); + let hash = format!("{hash:x}"); + eprintln!("{hash} *{asset_name}"); + + download_info.insert( + platform, + ManifestDownloadInfo { + url, + checksum: hash, + bin_dir: base_download_info + .bin_dir + .as_ref() + .or(base_info.bin_dir.as_ref()) + .cloned(), + bin: base_download_info + .bin + .as_ref() + .or(base_info.bin.as_ref()) + .map(|s| replace_vars(s, package, version, platform)), + }, + ); + buf.clear(); + } + if download_info.is_empty() { + eprintln!("no release asset for {package} {version}"); + continue; + } + if !base_info.prefer_linux_gnu { + // compact manifest + if download_info.contains_key(&HostPlatform::x86_64_linux_gnu) + && download_info.contains_key(&HostPlatform::x86_64_linux_musl) + { + download_info.remove(&HostPlatform::x86_64_linux_gnu); + } + if download_info.contains_key(&HostPlatform::aarch64_linux_gnu) + && download_info.contains_key(&HostPlatform::aarch64_linux_musl) + { + download_info.remove(&HostPlatform::aarch64_linux_gnu); + } + } + has_build_metadata |= !semver_version.build.is_empty(); + if semver_version.pre.is_empty() { + semver_versions.insert(semver_version.clone()); + } + manifests.insert( + Reverse(semver_version.clone().into()), + Manifest { + version: semver_version.into(), + download_info, + }, + ); + } + if has_build_metadata { + eprintln!( + "omitting patch/minor version is not supported yet for package with build metadata" + ); + } else if !semver_versions.is_empty() { + let mut prev_version = semver_versions.iter().next().unwrap(); + for version in &semver_versions { + if !(version.major == 0 && version.minor == 0) { + manifests.insert( + Reverse(Version::new(version.major, Some(version.minor))), + manifests[&Reverse(Version::from(version.clone()))].clone(), + ); + } + if version.major != 0 { + manifests.insert( + Reverse(Version::new(version.major, None)), + manifests[&Reverse(Version::from(version.clone()))].clone(), + ); + } + prev_version = version; + } + manifests.insert( + Reverse(Version::latest()), + manifests[&Reverse(Version::from(prev_version.clone()))].clone(), + ); + } + + if latest_only { + manifests.retain(|k, _| k.0 == Version::latest()); + } + + let mut buf = serde_json::to_vec_pretty(&manifests)?; + buf.push(b'\n'); + fs::write(manifest_path, buf)?; + + Ok(()) +} + +fn replace_vars(s: &str, package: &str, version: &str, platform: HostPlatform) -> String { + s.replace("${package}", package) + .replace("${tool}", package) + .replace("${rust_target}", platform.rust_target()) + .replace("${version}", version) + .replace("${exe}", platform.exe_suffix()) +} + +type Manifests = BTreeMap, Manifest>; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Version { + major: Option, + minor: Option, + patch: Option, + pre: semver::Prerelease, + build: semver::BuildMetadata, +} + +impl Version { + fn new(major: u64, minor: Option) -> Self { + Self { + major: Some(major), + minor, + patch: None, + pre: Default::default(), + build: Default::default(), + } + } + fn latest() -> Self { + Self { + major: None, + minor: None, + patch: None, + pre: Default::default(), + build: Default::default(), + } + } + fn to_semver(&self) -> Option { + Some(semver::Version { + major: self.major?, + minor: self.minor?, + patch: self.patch?, + pre: self.pre.clone(), + build: self.build.clone(), + }) + } +} +impl From for Version { + fn from(v: semver::Version) -> Self { + Self { + major: Some(v.major), + minor: Some(v.minor), + patch: Some(v.patch), + pre: v.pre, + build: v.build, + } + } +} +impl PartialOrd for Version { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for Version { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + fn convert(v: &Version) -> semver::Version { + semver::Version { + major: v.major.unwrap_or(u64::MAX), + minor: v.minor.unwrap_or(u64::MAX), + patch: v.patch.unwrap_or(u64::MAX), + pre: v.pre.clone(), + build: v.build.clone(), + } + } + convert(self).cmp(&convert(other)) + } +} +impl fmt::Display for Version { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + 'scope: { + let Some(major) = self.major else { + f.write_str("latest")?; + break 'scope; + }; + f.write_str(&major.to_string())?; + let Some(minor) = self.minor else { + break 'scope; + }; + f.write_str(".")?; + f.write_str(&minor.to_string())?; + let Some(patch) = self.patch else { + break 'scope; + }; + f.write_str(".")?; + f.write_str(&patch.to_string())?; + if !self.pre.is_empty() { + f.write_str("-")?; + f.write_str(&self.pre)?; + } + if !self.build.is_empty() { + f.write_str("+")?; + f.write_str(&self.build)?; + } + } + Ok(()) + } +} +impl FromStr for Version { + type Err = semver::Error; + fn from_str(s: &str) -> Result { + if s == "latest" { + return Ok(Self::latest()); + } + match s.parse::() { + Ok(v) => Ok(v.into()), + Err(e) => match s.parse::() { + Ok(v) => Ok(Self { + major: Some(v.major), + minor: v.minor, + patch: v.patch, + pre: Default::default(), + build: Default::default(), + }), + Err(_e) => Err(e), + }, + } + } +} +impl Serialize for Version { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + String::serialize(&self.to_string(), serializer) + } +} +impl<'de> Deserialize<'de> for Version { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::Error as _; + String::deserialize(deserializer)? + .parse() + .map_err(D::Error::custom) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct Manifest { + // TODO: only serialize version if key != version? + version: Version, + #[serde(flatten)] + download_info: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ManifestDownloadInfo { + url: String, + checksum: String, + /// Default to ${cargo_bin} + #[serde(skip_serializing_if = "Option::is_none")] + bin_dir: Option, + /// Default to ${tool}${exe} + #[serde(skip_serializing_if = "Option::is_none")] + bin: Option, +} + +#[derive(Debug, Deserialize)] +struct BaseManifest { + /// Link to the GitHub repository. + repository: String, + /// Prefix of release tag. + tag_prefix: String, + default_major_version: Option, + /// Asset name patterns. + asset_name: Option, + /// Directory where binary is installed. Default to `${cargo_bin}`. + bin_dir: Option, + /// Path to binary in archive. Default to `${tool}${exe}`. + bin: Option, + platform: BTreeMap, + /// Use glibc build if host_env is gnu. + #[serde(default)] + prefer_linux_gnu: bool, + version_range: Option, +} + +#[derive(Debug, Deserialize)] +struct BaseManifestPlatformInfo { + /// Asset name patterns. Default to the value at `BaseManifest::asset_name`. + asset_name: Option, + /// Directory where binary is installed. Default to the value at `BaseManifest::bin_dir`. + bin_dir: Option, + /// Path to binary in archive. Default to the value at `BaseManifest::bin`. + bin: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum StringOrArray { + String(String), + Array(Vec), +} + +impl StringOrArray { + fn as_slice(&self) -> &[String] { + match self { + Self::Array(v) => v, + Self::String(s) => slice::from_ref(s), + } + } +} + +/// GitHub Actions Runner supports Linux (x86_64, aarch64, arm), Windows (x86_64, aarch64), +/// and macOS (x86_64, aarch64). +/// https://github.com/actions/runner +/// https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners#supported-architectures-and-operating-systems-for-self-hosted-runners +/// +/// Note: +/// - Static-linked binaries compiled for linux-musl will also work on linux-gnu systems and are +/// usually preferred over linux-gnu binaries because they can avoid glibc version issues. +/// (rustc enables statically linking for linux-musl by default, except for mips.) +/// - Binaries compiled for x86_64 macOS will usually also work on aarch64 macOS. +/// - Binaries compiled for x86_64 Windows will usually also work on aarch64 Windows 11+. +/// - Ignore arm for now, as we need to consider the version and whether hard-float is supported. +/// https://github.com/rust-lang/rustup/pull/593 +/// https://github.com/cross-rs/cross/pull/1018 +/// Does it seem only armv7l is supported? +/// https://github.com/actions/runner/blob/6b9e8a6be411a6e63d5ccaf3c47e7b7622c5ec49/src/Misc/externals.sh#L174 +#[allow(non_camel_case_types)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +enum HostPlatform { + x86_64_linux_gnu, + x86_64_linux_musl, + x86_64_macos, + x86_64_windows, + aarch64_linux_gnu, + aarch64_linux_musl, + aarch64_macos, + aarch64_windows, +} + +impl HostPlatform { + fn rust_target(self) -> &'static str { + match self { + Self::x86_64_linux_gnu => "x86_64-unknown-linux-gnu", + Self::x86_64_linux_musl => "x86_64-unknown-linux-musl", + Self::x86_64_macos => "x86_64-apple-darwin", + Self::x86_64_windows => "x86_64-pc-windows-msvc", + Self::aarch64_linux_gnu => "aarch64-unknown-linux-gnu", + Self::aarch64_linux_musl => "aarch64-unknown-linux-musl", + Self::aarch64_macos => "aarch64-apple-darwin", + Self::aarch64_windows => "aarch64-pc-windows-msvc", + } + } + fn exe_suffix(self) -> &'static str { + match self { + Self::x86_64_windows | Self::aarch64_windows => ".exe", + _ => "", + } + } +} + +mod github { + use serde::Deserialize; + + // https://api.github.com/repos//releases + pub type Releases = Vec; + + // https://api.github.com/repos//releases/ + #[derive(Debug, Deserialize)] + pub struct Release { + pub tag_name: String, + pub prerelease: bool, + pub assets: Vec, + } + + #[derive(Debug, Deserialize)] + pub struct ReleaseAsset { + pub name: String, + pub content_type: String, + pub browser_download_url: String, + } +} diff --git a/tools/manifest.sh b/tools/manifest.sh new file mode 100755 index 00000000..bce3a24d --- /dev/null +++ b/tools/manifest.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' +cd "$(dirname "$0")"/.. + +# Update manifests +# +# USAGE: +# ./tools/manifest.sh [PACKAGE [VERSION_REQ]] + +if [[ $# -gt 0 ]]; then + cargo run --release -p install-action-internal-codegen -- "$@" + exit 0 +fi + +for manifest in tools/codegen/base/*.json; do + package="$(basename "${manifest%.*}")" + cargo run --release -p install-action-internal-codegen -- "${package}" latest +done