use anyhow::{bail, Context as _, Result}; use fs_err as fs; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ cmp::Reverse, collections::{BTreeMap, BTreeSet}, env, fmt, io::Read, path::{Path, PathBuf}, slice, str::FromStr, time::Duration, }; fn main() -> Result<()> { let args: Vec<_> = env::args().skip(1).collect(); if args.is_empty() || args.iter().any(|arg| arg.starts_with('-')) { println!( "USAGE: cargo run -p install-action-internal-codegen -r -- [VERSION_REQ]" ); std::process::exit(1); } let package = &args[0]; let workspace_root = &workspace_root(); let manifest_path = &workspace_root .join("manifests") .join(format!("{package}.json")); let download_cache_dir = &workspace_root.join("tools/codegen/tmp/cache").join(package); fs::create_dir_all(download_cache_dir)?; let base_info: BaseManifest = serde_json::from_slice(&fs::read( workspace_root .join("tools/codegen/base") .join(format!("{package}.json")), )?)?; let repo = base_info .repository .strip_prefix("https://github.com/") .context("repository must be starts with https://github.com/")?; eprintln!("downloading releases of https://github.com/{repo}"); let mut releases: github::Releases = vec![]; for page in 1.. { let per_page = 100; let mut r: github::Releases = download(&format!( "https://api.github.com/repos/{repo}/releases?per_page={per_page}&page={page}" ))? .into_json()?; if r.len() < per_page { releases.append(&mut r); break; } releases.append(&mut r); } let releases: Vec<_> = releases .iter() .filter_map(|release| { release .tag_name .strip_prefix(&base_info.tag_prefix) .map(|version| (version, release)) }) .collect(); let mut manifests: Manifests = Manifests::default(); let mut semver_versions = BTreeSet::new(); let mut has_build_metadata = false; let mut latest_only = false; if let Some(version_range) = &base_info.version_range { if version_range == "latest" { latest_only = true; } } if manifest_path.is_file() { match serde_json::from_slice(&fs::read(manifest_path)?) { Ok(m) => { manifests = m; for (k, manifest) in &mut manifests.map { let ManifestRef::Real(manifest) = manifest else { continue }; let version = &*k.0.to_string(); if let Some(template) = &manifests.template { for (platform, d) in &mut manifest.download_info { let template = &template.download_info[platform]; d.url = Some(template.url.replace("${version}", version)); d.bin_dir = template .bin_dir .as_ref() .map(|s| s.replace("${version}", version)); d.bin = template .bin .as_ref() .map(|s| s.replace("${version}", version)); } } } manifests.template = None; } Err(e) => eprintln!("failed to load old manifest: {e}"), } } let version_req: Option = match args.get(1) { _ if latest_only => { if args.get(1).map(String::as_str) == Some("latest") { if let Some(m) = manifests.map.first_key_value() { let version = match m.1 { ManifestRef::Ref { version } => version, ManifestRef::Real(_) => &m.0 .0, }; if !manifests.map.is_empty() && *version == releases.first().unwrap().0.parse()? { return Ok(()); } } } Some(format!("={}", releases.first().unwrap().0).parse()?) } None => match base_info.version_range { Some(version_range) => Some(version_range.parse()?), None => Some(">= 0.0.1".parse()?), // HACK: ignore pre-releases }, Some(version_req) => { for version in manifests.map.keys() { let Some(semver_version) = version.0.to_semver() else { continue; }; has_build_metadata |= !semver_version.build.is_empty(); if semver_version.pre.is_empty() { semver_versions.insert(semver_version.clone()); } } let req = if version_req == "latest" { if manifests.map.is_empty() { format!("={}", releases.first().unwrap().0).parse()? } else { format!(">{}", semver_versions.last().unwrap()).parse()? } } else { version_req.parse()? }; eprintln!("update manifest for versions '{req}'"); Some(req) } }; let mut buf = vec![]; for &(version, release) in &releases { let mut semver_version = version.parse::(); if semver_version.is_err() { if let Some(default_major_version) = &base_info.default_major_version { semver_version = format!("{default_major_version}.{version}").parse(); } } let Ok(semver_version) = semver_version else { continue; }; if let Some(version_req) = &version_req { if !version_req.matches(&semver_version) { continue; } } let mut download_info = BTreeMap::new(); for (&platform, base_download_info) in &base_info.platform { let asset_names = base_download_info .asset_name .as_ref() .or(base_info.asset_name.as_ref()) .with_context(|| format!("asset_name is needed for {package} on {platform:?}"))? .as_slice() .iter() .map(|asset_name| replace_vars(asset_name, package, version, platform)) .collect::>>()?; let (url, asset_name) = match asset_names.iter().find_map(|asset_name| { release .assets .iter() .find(|asset| asset.name == *asset_name) .map(|asset| (asset, asset_name)) }) { Some((asset, asset_name)) => { (asset.browser_download_url.clone(), asset_name.clone()) } None => { eprintln!("no asset '{asset_names:?}' for host platform '{platform:?}'"); continue; } }; eprintln!("downloading {url} for checksum"); let download_cache = download_cache_dir.join(format!( "{version}-{platform:?}-{}", Path::new(&url).file_name().unwrap().to_str().unwrap() )); if download_cache.is_file() { eprintln!("{url} is already downloaded"); fs::File::open(download_cache)?.read_to_end(&mut buf)?; } else { download(&url)?.into_reader().read_to_end(&mut buf)?; eprintln!("downloaded complete"); fs::write(download_cache, &buf)?; } eprintln!("getting sha256 hash for {url}"); let hash = Sha256::digest(&buf); let hash = format!("{hash:x}"); eprintln!("{hash} *{asset_name}"); download_info.insert( platform, ManifestDownloadInfo { url: Some(url), checksum: hash, bin_dir: base_download_info .bin_dir .as_ref() .or(base_info.bin_dir.as_ref()) .cloned(), bin: base_download_info .bin .as_ref() .or(base_info.bin.as_ref()) .map(|s| replace_vars(s, package, version, platform)) .transpose()?, }, ); buf.clear(); } if download_info.is_empty() { eprintln!("no release asset for {package} {version}"); continue; } if !base_info.prefer_linux_gnu { // compact manifest if download_info.contains_key(&HostPlatform::x86_64_linux_gnu) && download_info.contains_key(&HostPlatform::x86_64_linux_musl) { download_info.remove(&HostPlatform::x86_64_linux_gnu); } if download_info.contains_key(&HostPlatform::aarch64_linux_gnu) && download_info.contains_key(&HostPlatform::aarch64_linux_musl) { download_info.remove(&HostPlatform::aarch64_linux_gnu); } } has_build_metadata |= !semver_version.build.is_empty(); if semver_version.pre.is_empty() { semver_versions.insert(semver_version.clone()); } manifests.map.insert( Reverse(semver_version.clone().into()), ManifestRef::Real(Manifest { download_info }), ); } if has_build_metadata { eprintln!( "omitting patch/minor version is not supported yet for package with build metadata" ); } else if !semver_versions.is_empty() { let mut prev_version = semver_versions.iter().next().unwrap(); for version in &semver_versions { if !(version.major == 0 && version.minor == 0) { manifests.map.insert( Reverse(Version::omitted(version.major, Some(version.minor))), ManifestRef::Ref { version: version.clone().into(), }, ); } if version.major != 0 { manifests.map.insert( Reverse(Version::omitted(version.major, None)), ManifestRef::Ref { version: version.clone().into(), }, ); } prev_version = version; } manifests.map.insert( Reverse(Version::latest()), ManifestRef::Ref { version: prev_version.clone().into(), }, ); } if latest_only { let ManifestRef::Ref { version } = manifests.map.first_key_value().unwrap().1.clone() else { unreachable!() }; manifests .map .retain(|k, _| k.0 == Version::latest() || k.0 == version); } let original_manifests = manifests.clone(); let mut template = Some(ManifestTemplate { download_info: BTreeMap::new(), }); 'outer: for (version, manifest) in &mut manifests.map { let ManifestRef::Real(manifest) = manifest else { continue }; let version = &*version.0.to_string(); let t = template.as_mut().unwrap(); for (platform, d) in &mut manifest.download_info { let template_url = d.url.take().unwrap().replace(version, "${version}"); let template_bin_dir = d.bin_dir.take().map(|s| s.replace(version, "${version}")); let template_bin = d.bin.take().map(|s| s.replace(version, "${version}")); if let Some(d) = t.download_info.get(platform) { if template_url != d.url || template_bin_dir != d.bin_dir || template_bin != d.bin { template = None; break 'outer; } } else { t.download_info.insert( *platform, ManifestTemplateDownloadInfo { url: template_url, bin_dir: template_bin_dir, bin: template_bin, }, ); } } } if template.is_none() { manifests = original_manifests; } else { manifests.template = template; } let mut buf = serde_json::to_vec_pretty(&manifests)?; buf.push(b'\n'); fs::write(manifest_path, buf)?; Ok(()) } fn workspace_root() -> PathBuf { let mut dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); dir.pop(); // codegen dir.pop(); // tools dir } fn replace_vars(s: &str, package: &str, version: &str, platform: HostPlatform) -> Result { let s = s .replace("${package}", package) .replace("${tool}", package) .replace("${rust_target}", platform.rust_target()) .replace("${version}", version) .replace("${exe}", platform.exe_suffix()); if s.contains('$') { bail!("variable not fully replaced: '{s}'"); } Ok(s) } fn download(url: &str) -> Result { let token = env::var("INTERNAL_CODEGEN_GH_PAT").ok(); let mut retry = 0; let mut last_error = None; while retry < 5 { let mut req = ureq::get(url); if let Some(token) = &token { req = req.set("Authorization", token); } match req.call() { Ok(res) => return Ok(res), Err(e) => last_error = Some(e), } retry += 1; eprintln!("download failed; retrying ({retry}/5)"); std::thread::sleep(Duration::from_secs(retry)); } Err(last_error.unwrap().into()) } #[derive(Debug, Clone, PartialEq, Eq)] struct Version { major: Option, minor: Option, patch: Option, pre: semver::Prerelease, build: semver::BuildMetadata, } impl Version { fn omitted(major: u64, minor: Option) -> Self { Self { major: Some(major), minor, patch: None, pre: Default::default(), build: Default::default(), } } fn latest() -> Self { Self { major: None, minor: None, patch: None, pre: Default::default(), build: Default::default(), } } fn to_semver(&self) -> Option { Some(semver::Version { major: self.major?, minor: self.minor?, patch: self.patch?, pre: self.pre.clone(), build: self.build.clone(), }) } } impl From for Version { fn from(v: semver::Version) -> Self { Self { major: Some(v.major), minor: Some(v.minor), patch: Some(v.patch), pre: v.pre, build: v.build, } } } impl PartialOrd for Version { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } impl Ord for Version { fn cmp(&self, other: &Self) -> std::cmp::Ordering { fn convert(v: &Version) -> semver::Version { semver::Version { major: v.major.unwrap_or(u64::MAX), minor: v.minor.unwrap_or(u64::MAX), patch: v.patch.unwrap_or(u64::MAX), pre: v.pre.clone(), build: v.build.clone(), } } convert(self).cmp(&convert(other)) } } impl fmt::Display for Version { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let Some(major) = self.major else { f.write_str("latest")?; return Ok(()); }; f.write_str(&major.to_string())?; let Some(minor) = self.minor else { return Ok(()); }; f.write_str(".")?; f.write_str(&minor.to_string())?; let Some(patch) = self.patch else { return Ok(()); }; f.write_str(".")?; f.write_str(&patch.to_string())?; if !self.pre.is_empty() { f.write_str("-")?; f.write_str(&self.pre)?; } if !self.build.is_empty() { f.write_str("+")?; f.write_str(&self.build)?; } Ok(()) } } impl FromStr for Version { type Err = semver::Error; fn from_str(s: &str) -> Result { if s == "latest" { return Ok(Self::latest()); } match s.parse::() { Ok(v) => Ok(v.into()), Err(e) => match s.parse::() { Ok(v) => Ok(Self { major: Some(v.major), minor: v.minor, patch: v.patch, pre: Default::default(), build: Default::default(), }), Err(_e) => Err(e), }, } } } impl Serialize for Version { fn serialize(&self, serializer: S) -> Result where S: serde::Serializer, { String::serialize(&self.to_string(), serializer) } } impl<'de> Deserialize<'de> for Version { fn deserialize(deserializer: D) -> Result where D: serde::Deserializer<'de>, { use serde::de::Error as _; String::deserialize(deserializer)? .parse() .map_err(D::Error::custom) } } #[derive(Debug, Clone, Default, Serialize, Deserialize)] struct Manifests { template: Option, #[serde(flatten)] map: BTreeMap, ManifestRef>, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(untagged)] enum ManifestRef { Ref { version: Version }, Real(Manifest), } #[derive(Debug, Clone, Serialize, Deserialize)] struct Manifest { #[serde(flatten)] download_info: BTreeMap, } #[derive(Debug, Clone, Serialize, Deserialize)] struct ManifestDownloadInfo { #[serde(skip_serializing_if = "Option::is_none")] url: Option, checksum: String, /// Default to ${cargo_bin} #[serde(skip_serializing_if = "Option::is_none")] bin_dir: Option, /// Default to ${tool}${exe} #[serde(skip_serializing_if = "Option::is_none")] bin: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] struct ManifestTemplate { #[serde(flatten)] download_info: BTreeMap, } #[derive(Debug, Clone, Serialize, Deserialize)] struct ManifestTemplateDownloadInfo { url: String, /// Default to ${cargo_bin} #[serde(skip_serializing_if = "Option::is_none")] bin_dir: Option, /// Default to ${tool}${exe} #[serde(skip_serializing_if = "Option::is_none")] bin: Option, } #[derive(Debug, Deserialize)] struct BaseManifest { /// Link to the GitHub repository. repository: String, /// Prefix of release tag. tag_prefix: String, default_major_version: Option, /// Asset name patterns. asset_name: Option, /// Directory where binary is installed. Default to `${cargo_bin}`. bin_dir: Option, /// Path to binary in archive. Default to `${tool}${exe}`. bin: Option, platform: BTreeMap, /// Use glibc build if host_env is gnu. #[serde(default)] prefer_linux_gnu: bool, version_range: Option, } #[derive(Debug, Deserialize)] struct BaseManifestPlatformInfo { /// Asset name patterns. Default to the value at `BaseManifest::asset_name`. asset_name: Option, /// Directory where binary is installed. Default to the value at `BaseManifest::bin_dir`. bin_dir: Option, /// Path to binary in archive. Default to the value at `BaseManifest::bin`. bin: Option, } #[derive(Debug, Clone, Deserialize)] #[serde(untagged)] enum StringOrArray { String(String), Array(Vec), } impl StringOrArray { fn as_slice(&self) -> &[String] { match self { Self::Array(v) => v, Self::String(s) => slice::from_ref(s), } } } /// GitHub Actions Runner supports Linux (x86_64, aarch64, arm), Windows (x86_64, aarch64), /// and macOS (x86_64, aarch64). /// https://github.com/actions/runner /// https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners#supported-architectures-and-operating-systems-for-self-hosted-runners /// /// Note: /// - Static-linked binaries compiled for linux-musl will also work on linux-gnu systems and are /// usually preferred over linux-gnu binaries because they can avoid glibc version issues. /// (rustc enables statically linking for linux-musl by default, except for mips.) /// - Binaries compiled for x86_64 macOS will usually also work on aarch64 macOS. /// - Binaries compiled for x86_64 Windows will usually also work on aarch64 Windows 11+. /// - Ignore arm for now, as we need to consider the version and whether hard-float is supported. /// https://github.com/rust-lang/rustup/pull/593 /// https://github.com/cross-rs/cross/pull/1018 /// Does it seem only armv7l is supported? /// https://github.com/actions/runner/blob/6b9e8a6be411a6e63d5ccaf3c47e7b7622c5ec49/src/Misc/externals.sh#L174 #[allow(non_camel_case_types)] #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] enum HostPlatform { x86_64_linux_gnu, x86_64_linux_musl, x86_64_macos, x86_64_windows, aarch64_linux_gnu, aarch64_linux_musl, aarch64_macos, aarch64_windows, } impl HostPlatform { fn rust_target(self) -> &'static str { match self { Self::x86_64_linux_gnu => "x86_64-unknown-linux-gnu", Self::x86_64_linux_musl => "x86_64-unknown-linux-musl", Self::x86_64_macos => "x86_64-apple-darwin", Self::x86_64_windows => "x86_64-pc-windows-msvc", Self::aarch64_linux_gnu => "aarch64-unknown-linux-gnu", Self::aarch64_linux_musl => "aarch64-unknown-linux-musl", Self::aarch64_macos => "aarch64-apple-darwin", Self::aarch64_windows => "aarch64-pc-windows-msvc", } } fn exe_suffix(self) -> &'static str { match self { Self::x86_64_windows | Self::aarch64_windows => ".exe", _ => "", } } } mod github { use serde::Deserialize; // https://api.github.com/repos//releases pub type Releases = Vec; // https://api.github.com/repos//releases/ #[derive(Debug, Deserialize)] pub struct Release { pub tag_name: String, pub prerelease: bool, pub assets: Vec, } #[derive(Debug, Deserialize)] pub struct ReleaseAsset { pub name: String, pub content_type: String, pub browser_download_url: String, } }