From 5626f10fce0806a71c6176169714655d060ecbe6 Mon Sep 17 00:00:00 2001 From: Mark Rousskov Date: Sun, 9 Aug 2026 12:15:02 +0300 Subject: [PATCH] Added org github tokens to check untracked private repositories --- .github/workflows/check-untracked-repos.yml | 14 +++ src/api/github.rs | 112 ++++++++++++++++++-- src/ci.rs | 4 +- src/sync/github/api/mod.rs | 2 +- src/sync/github/api/tokens.rs | 51 +++++++-- src/sync/github/mod.rs | 1 + src/sync/mod.rs | 2 + 7 files changed, 166 insertions(+), 20 deletions(-) diff --git a/.github/workflows/check-untracked-repos.yml b/.github/workflows/check-untracked-repos.yml index 25fc78e3a..093a9aa46 100644 --- a/.github/workflows/check-untracked-repos.yml +++ b/.github/workflows/check-untracked-repos.yml @@ -36,8 +36,22 @@ jobs: - name: Install Rust stable uses: ./.github/actions/setup-rust + # Used to detect private repos in allowed-github-orgs + - name: Generate GitHub App tokens + uses: ./.github/actions/generate-tokens + id: generate-tokens + with: + app-id: ${{ secrets.SYNC_TEAM_GH_APP_ID }} + private-key: ${{ secrets.SYNC_TEAM_GH_APP_PRIVATE_KEY }} + - name: Check untracked repositories id: check + env: + GITHUB_TOKEN_RUST_LANG: ${{ steps.generate-tokens.outputs.rust-lang-token }} + GITHUB_TOKEN_RUST_LANG_DEPRECATED: ${{ steps.generate-tokens.outputs.rust-lang-deprecated-token }} + GITHUB_TOKEN_RUST_LANG_NURSERY: ${{ steps.generate-tokens.outputs.rust-lang-nursery-token }} + GITHUB_TOKEN_RUST_ANALYZER: ${{ steps.generate-tokens.outputs.rust-analyzer-token }} + GITHUB_TOKEN_RUST_DEV_TOOLS: ${{ steps.generate-tokens.outputs.rust-dev-tools-token }} run: | cargo build --release diff --git a/src/api/github.rs b/src/api/github.rs index 33f95c06e..6d330375d 100644 --- a/src/api/github.rs +++ b/src/api/github.rs @@ -1,3 +1,4 @@ +use crate::sync::GitHubTokens; use crate::sync::utils::ResponseExt; use anyhow::{Context, Error, bail}; use base64::Engine; @@ -6,6 +7,7 @@ use chrono::{DateTime, Duration, Utc}; use reqwest::header::{self, HeaderValue}; use reqwest::{Client, ClientBuilder, RequestBuilder}; use reqwest::{Method, StatusCode}; +use secrecy::ExposeSecret; use std::borrow::Cow; use std::collections::HashMap; @@ -40,6 +42,7 @@ struct GraphNodes { pub(crate) struct GitHubApi { http: Client, token: Option, + org_tokens: Option, } impl GitHubApi { @@ -50,12 +53,25 @@ impl GitHubApi { .build() .unwrap(), token: std::env::var(TOKEN_VAR).ok(), + org_tokens: None, + } + } + + pub(crate) fn new_with_org_tokens() -> Self { + GitHubApi { + http: ClientBuilder::new() + .user_agent(crate::USER_AGENT) + .build() + .unwrap(), + token: std::env::var(TOKEN_VAR).ok(), + org_tokens: Some(GitHubTokens::from_env_org_tokens_only()), } } fn prepare( &self, require_auth: bool, + org: Option<&str>, method: Method, url: &str, ) -> Result { @@ -69,7 +85,16 @@ impl GitHubApi { } let mut req = self.http.request(method, url.as_ref()); - if let Some(token) = &self.token { + let token = match org { + Some(org) => self + .org_tokens + .as_ref() + .and_then(|tokens| tokens.get_organization_token(org).ok()) + .map(|token| token.expose_secret()), + None => self.token.as_deref(), + }; + + if let Some(token) = token { req = req.header( header::AUTHORIZATION, HeaderValue::from_str(&format!("token {token}"))?, @@ -89,7 +114,7 @@ impl GitHubApi { variables: V, } let res: GraphResult = self - .prepare(true, Method::POST, "graphql")? + .prepare(true, None, Method::POST, "graphql")? .json(&Request { query, variables }) .send() .await? @@ -113,7 +138,7 @@ impl GitHubApi { } pub(crate) async fn user(&self, login: &str) -> Result { - self.prepare(false, Method::GET, &format!("users/{login}"))? + self.prepare(false, None, Method::GET, &format!("users/{login}"))? .send() .await? .error_for_status()? @@ -121,12 +146,12 @@ impl GitHubApi { .await } - pub(crate) async fn get(&self, url: &str) -> Result + pub(crate) async fn get(&self, org: Option<&str>, url: &str) -> Result where T: serde::de::DeserializeOwned, { loop { - let response = self.prepare(false, Method::GET, url)?.send().await?; + let response = self.prepare(false, org, Method::GET, url)?.send().await?; let status = response.status(); if status != StatusCode::OK { @@ -378,7 +403,7 @@ query($query: String!, $issueLimit: Int!, $commentLimit: Int!) { items: Vec, } - let response: Response = self.get(&format!("search/commits?q=author:{username}+org:{org}&sort=author-date&order=desc&per_page={limit}")).await?; + let response: Response = self.get(None, &format!("search/commits?q=author:{username}+org:{org}&sort=author-date&order=desc&per_page={limit}")).await?; Ok(response .items .into_iter() @@ -419,3 +444,78 @@ pub struct CommitInfo { pub repo_name: String, pub created_at: DateTime, } + +#[cfg(test)] +mod tests { + use super::*; + use secrecy::SecretString; + + #[test] + fn prepare_uses_organization_token() { + let github = github_with_org_tokens(HashMap::from([( + "rust-lang".to_string(), + SecretString::from("organization-token"), + )])); + + let request = github + .prepare( + false, + Some("rust-lang"), + Method::GET, + "orgs/rust-lang/repos", + ) + .unwrap() + .build() + .unwrap(); + + assert_eq!( + request.headers().get(header::AUTHORIZATION).unwrap(), + "token organization-token" + ); + } + + #[test] + fn prepare_without_organization_token_is_unauthenticated() { + let github = github_with_org_tokens(HashMap::new()); + + let request = github + .prepare( + false, + Some("rust-lang"), + Method::GET, + "orgs/rust-lang/repos", + ) + .unwrap() + .build() + .unwrap(); + + assert!(!request.headers().contains_key(header::AUTHORIZATION)); + } + + #[test] + fn prepare_without_organization_uses_default_token() { + let github = github_with_org_tokens(HashMap::new()); + + let request = github + .prepare(false, None, Method::GET, "users/rust-lang-owner") + .unwrap() + .build() + .unwrap(); + + assert_eq!( + request.headers().get(header::AUTHORIZATION).unwrap(), + "token default-token" + ); + } + + fn github_with_org_tokens(org_tokens: HashMap) -> GitHubApi { + GitHubApi { + http: ClientBuilder::new().build().unwrap(), + token: Some("default-token".to_string()), + org_tokens: Some(GitHubTokens::App { + org_tokens, + enterprise_client_ctx: None, + }), + } + } +} diff --git a/src/ci.rs b/src/ci.rs index 4b6d40aac..642e51fa3 100644 --- a/src/ci.rs +++ b/src/ci.rs @@ -202,7 +202,7 @@ pub async fn check_untracked_repos( data_dir: &Path, create_missing: bool, ) -> anyhow::Result { - let github = crate::api::github::GitHubApi::new(); + let github = crate::api::github::GitHubApi::new_with_org_tokens(); // Get allowed GitHub organizations from config instead of hardcoding let orgs_to_monitor: Vec<&str> = data @@ -276,7 +276,7 @@ async fn fetch_all_github_repos( let url = format!("orgs/{}/repos?per_page=100&page={}", org, page); let repos: Vec = github - .get(&url) + .get(Some(org), &url) .await .with_context(|| format!("Failed to fetch repos for org: {}", org))?; diff --git a/src/sync/github/api/mod.rs b/src/sync/github/api/mod.rs index d39c85990..bdd622197 100644 --- a/src/sync/github/api/mod.rs +++ b/src/sync/github/api/mod.rs @@ -19,11 +19,11 @@ use serde::{Deserialize, de::DeserializeOwned}; use std::collections::BTreeSet; use std::fmt; use thiserror::Error; -use tokens::GitHubTokens; use url::GitHubUrl; use crate::sync::Config; pub(crate) use read::{GitHubApiRead, GithubRead}; +pub(crate) use tokens::GitHubTokens; pub(crate) use write::GitHubWrite; #[derive(Debug, Error)] diff --git a/src/sync/github/api/tokens.rs b/src/sync/github/api/tokens.rs index 09d09fd01..a9607dbfd 100644 --- a/src/sync/github/api/tokens.rs +++ b/src/sync/github/api/tokens.rs @@ -43,7 +43,7 @@ pub enum GitHubTokens { /// The token has to be available for the whole duration of the process. org_tokens: HashMap, /// Context for using enterprise GitHub App. - enterprise_client_ctx: EnterpriseAppCtx, + enterprise_client_ctx: Option, }, /// One token for all API calls (used with Personal Access Token). Pat(SecretString), @@ -55,13 +55,7 @@ impl GitHubTokens { /// Parses environment variables in the format GITHUB_TOKEN_{ORG_NAME} /// to retrieve GitHub tokens. pub async fn from_env(config: &Config) -> anyhow::Result { - let mut tokens = HashMap::new(); - - for (key, value) in std::env::vars() { - if let Some(org_name) = org_name_from_env_var(&key) { - tokens.insert(org_name, SecretString::from(value)); - } - } + let tokens = collect_org_envs(); if tokens.is_empty() { let pat_token = std::env::var("GITHUB_TOKEN") @@ -136,15 +130,26 @@ impl GitHubTokens { Ok(GitHubTokens::App { org_tokens: tokens, - enterprise_client_ctx: EnterpriseAppCtx { + enterprise_client_ctx: Some(EnterpriseAppCtx { enterprise_token, org_tokens: enterprise_org_tokens, enterprise_name, - }, + }), }) } } + pub fn from_env_org_tokens_only() -> Self { + GitHubTokens::App { + org_tokens: collect_org_envs(), + enterprise_client_ctx: None, + } + } + + pub fn get_organization_token(&self, org: &str) -> anyhow::Result<&SecretString> { + Self::get_token_for_org(self, org, &TokenType::Organization) + } + /// Get a token for a GitHub organization. /// Return an error if not present. pub fn get_token_for_org( @@ -163,6 +168,10 @@ impl GitHubTokens { ) }), TokenType::EnterpriseOrganization => { + let enterprise_client_ctx = enterprise_client_ctx + .as_ref() + .context("No enterprise GitHub App is configured")?; + enterprise_client_ctx.org_tokens.get(org).with_context(|| { format!( "failed to get the GitHub token environment variable for organization `{org}` for the enterprise GH app" @@ -170,6 +179,10 @@ impl GitHubTokens { }) } TokenType::Enterprise => { + let enterprise_client_ctx = enterprise_client_ctx + .as_ref() + .context("No enterprise GitHub App is configured")?; + Ok(&enterprise_client_ctx.enterprise_token) } }, @@ -183,7 +196,11 @@ impl GitHubTokens { GitHubTokens::App { enterprise_client_ctx, .. - } => Ok(enterprise_client_ctx.enterprise_name.as_str()), + } => Ok(enterprise_client_ctx + .as_ref() + .context("No enterprise GitHub App is configured")? + .enterprise_name + .as_str()), GitHubTokens::Pat(_) => Err(anyhow::anyhow!( "No enterprise is configured when using a PAT" )), @@ -191,6 +208,18 @@ impl GitHubTokens { } } +fn collect_org_envs() -> HashMap { + let mut tokens = HashMap::new(); + + for (key, value) in std::env::vars() { + if let Some(org_name) = org_name_from_env_var(&key) { + tokens.insert(org_name, SecretString::from(value)); + } + } + + tokens +} + fn org_name_from_env_var(env_var: &str) -> Option { env_var.strip_prefix("GITHUB_TOKEN_").map(|org| { // GitHub environment variables can't contain `-`, while GitHub organizations diff --git a/src/sync/github/mod.rs b/src/sync/github/mod.rs index 7fd165584..ace25e932 100644 --- a/src/sync/github/mod.rs +++ b/src/sync/github/mod.rs @@ -2,6 +2,7 @@ mod api; #[cfg(test)] mod tests; +pub(crate) use self::api::GitHubTokens; pub(crate) use self::api::{GitHubApiRead, GitHubWrite, HttpClient}; use self::api::{TeamPrivacy, TeamRole}; use crate::schema; diff --git a/src/sync/mod.rs b/src/sync/mod.rs index cebbd878b..2c5955c02 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -6,6 +6,8 @@ pub mod team_api; pub mod utils; mod zulip; +pub(crate) use github::GitHubTokens; + use std::collections::BTreeSet; use anyhow::Context;