Skip to content

Sync organization members #1868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,29 @@ permissions-bools = [
"dev-desktop",
"sync-team-confirmation",
]

# GitHub accounts that are allowed to stay in the orgs,
# even if they may not be members of any team.
allowed-org-members = [
Copy link
Member

@Kobzol Kobzol Jul 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: allowed makes it sound like only these org members may be a part of an org :D Which of course doesn't make sense, and the comment does clarify it. But I don't have a great alternative. Maybe just special-org-members? But it doesn't really matter.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like special-org-members

# Bots.
"bors",
"craterbot",
"rust-embedded-bot",
"rust-highfive",
"rust-lang-core-team-agenda-bot",
"rust-lang-glacier-bot",
"rust-lang-owner",
"rust-timer",
"rustbot",

# Infra admins.
"jdno",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, we start having a bunch of places where we hardcode infra-admins. Maybe it would be better if we loaded the actual contents of the infra-admins team (so that we hardcode only the team in code, but not its members) in code and ignored them?

"marcoieni",
"Mark-Simulacrum",
"pietroalbini",

# Ferrous systems employees who manage the github sponsors
# for rust-analyzer.
"missholmes",
"skade",
]
7 changes: 7 additions & 0 deletions rust_team_data/src/v1.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
use std::collections::BTreeSet;

use indexmap::IndexMap;
use serde::{Deserialize, Serialize};

pub static BASE_URL: &str = "https://team-api.infra.rust-lang.org/v1";

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we need to expose this to the JSON API? It's an internal implementation detail. Can't you just pass &Config from team to sync-team in run_sync_team?

pub struct Config {
pub allowed_org_members: BTreeSet<String>,
}

#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "snake_case")]
pub enum TeamKind {
Expand Down
8 changes: 7 additions & 1 deletion src/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ pub(crate) use crate::permissions::Permissions;
use anyhow::{bail, format_err, Error};
use serde::de::{Deserialize, Deserializer};
use serde_untagged::UntaggedEnumVisitor;
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeSet, HashMap, HashSet};

#[derive(serde_derive::Deserialize, Debug)]
#[serde(deny_unknown_fields, rename_all = "kebab-case")]
Expand All @@ -12,6 +12,8 @@ pub(crate) struct Config {
allowed_github_orgs: HashSet<String>,
permissions_bors_repos: HashSet<String>,
permissions_bools: HashSet<String>,
// Use a BTreeSet for consistent ordering in tests
allowed_org_members: BTreeSet<String>,
}

impl Config {
Expand All @@ -30,6 +32,10 @@ impl Config {
pub(crate) fn permissions_bools(&self) -> &HashSet<String> {
&self.permissions_bools
}

pub(crate) fn allowed_org_members(&self) -> &BTreeSet<String> {
&self.allowed_org_members
}
}

// This is an enum to allow two kinds of values for the email field:
Expand Down
10 changes: 10 additions & 0 deletions src/static_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ impl<'a> Generator<'a> {
self.generate_rfcbot()?;
self.generate_zulip_map()?;
self.generate_people()?;
self.generate_config()?;
self.generate_index_html()?;
Ok(())
}
Expand Down Expand Up @@ -458,6 +459,15 @@ impl<'a> Generator<'a> {
Ok(())
}

fn generate_config(&self) -> Result<(), Error> {
let config = v1::Config {
allowed_org_members: self.data.config().allowed_org_members().clone(),
};

self.add("v1/config.json", &config)?;
Ok(())
}

fn generate_index_html(&self) -> Result<(), Error> {
const CONTENT: &[u8] = b"\
<!DOCTYPE html>\n\
Expand Down
23 changes: 23 additions & 0 deletions sync-team/src/github/api/read.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ pub(crate) trait GithubRead {
/// Get the owners of an org
fn org_owners(&self, org: &str) -> anyhow::Result<HashSet<u64>>;

/// Get the members of an org
fn org_members(&self, org: &str) -> anyhow::Result<HashMap<u64, String>>;

/// Get all teams associated with a org
///
/// Returns a list of tuples of team name and slug
Expand Down Expand Up @@ -119,6 +122,26 @@ impl GithubRead for GitHubApiRead {
Ok(owners)
}

fn org_members(&self, org: &str) -> anyhow::Result<HashMap<u64, String>> {
#[derive(serde::Deserialize, Eq, PartialEq, Hash)]
struct User {
id: u64,
login: String,
}
let mut members = HashMap::new();
self.client.rest_paginated(
&Method::GET,
&GitHubUrl::orgs(org, "members")?,
|resp: Vec<User>| {
for user in resp {
members.insert(user.id, user.login);
}
Ok(())
},
)?;
Ok(members)
}

fn org_teams(&self, org: &str) -> anyhow::Result<Vec<(String, String)>> {
let mut teams = Vec::new();

Expand Down
12 changes: 12 additions & 0 deletions sync-team/src/github/api/write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,18 @@ impl GitHubWrite {
Ok(())
}

/// Remove a member from an org
pub(crate) fn remove_gh_member_from_org(&self, org: &str, user: &str) -> anyhow::Result<()> {
debug!("Removing user {user} from org {org}");
if !self.dry_run {
let method = Method::DELETE;
let url = GitHubUrl::orgs(org, &format!("members/{user}"))?;
let resp = self.client.req(method.clone(), &url)?.send()?;
allow_not_found(resp, method, url.url())?;
}
Ok(())
}

/// Remove a collaborator from a repo
pub(crate) fn remove_collaborator_from_repo(
&self,
Expand Down
133 changes: 130 additions & 3 deletions sync-team/src/github/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use self::api::{BranchProtectionOp, TeamPrivacy, TeamRole};
use crate::github::api::{GithubRead, Login, PushAllowanceActor, RepoPermission, RepoSettings};
use log::debug;
use rust_team_data::v1::{Bot, BranchProtectionMode, MergeBot};
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt::Write;

pub(crate) use self::api::{GitHubApiRead, GitHubWrite, HttpClient};
Expand All @@ -18,8 +18,9 @@ pub(crate) fn create_diff(
github: Box<dyn GithubRead>,
teams: Vec<rust_team_data::v1::Team>,
repos: Vec<rust_team_data::v1::Repo>,
config: rust_team_data::v1::Config,
) -> anyhow::Result<Diff> {
let github = SyncGitHub::new(github, teams, repos)?;
let github = SyncGitHub::new(github, teams, repos, config)?;
github.diff_all()
}

Expand All @@ -29,15 +30,18 @@ struct SyncGitHub {
github: Box<dyn GithubRead>,
teams: Vec<rust_team_data::v1::Team>,
repos: Vec<rust_team_data::v1::Repo>,
config: rust_team_data::v1::Config,
usernames_cache: HashMap<u64, String>,
org_owners: HashMap<OrgName, HashSet<u64>>,
org_members: HashMap<OrgName, HashMap<u64, String>>,
}

impl SyncGitHub {
pub(crate) fn new(
github: Box<dyn GithubRead>,
teams: Vec<rust_team_data::v1::Team>,
repos: Vec<rust_team_data::v1::Repo>,
config: rust_team_data::v1::Config,
) -> anyhow::Result<Self> {
debug!("caching mapping between user ids and usernames");
let users = teams
Expand All @@ -60,30 +64,112 @@ impl SyncGitHub {
.collect::<HashSet<_>>();

let mut org_owners = HashMap::new();
let mut org_members = HashMap::new();

for org in &orgs {
org_owners.insert((*org).to_string(), github.org_owners(org)?);
org_members.insert((*org).to_string(), github.org_members(org)?);
}

Ok(SyncGitHub {
github,
teams,
repos,
config,
usernames_cache,
org_owners,
org_members,
})
}

pub(crate) fn diff_all(&self) -> anyhow::Result<Diff> {
let team_diffs = self.diff_teams()?;
let repo_diffs = self.diff_repos()?;
let org_membership_diffs = self.diff_org_memberships()?;

Ok(Diff {
team_diffs,
repo_diffs,
org_membership_diffs,
})
}

/// Collect all org members from the respective teams
fn get_org_members_from_teams(&self) -> HashMap<OrgName, HashSet<u64>> {
let mut org_team_members: HashMap<OrgName, HashSet<u64>> = HashMap::new();

for team in &self.teams {
if let Some(gh) = &team.github {
for toml_gh_team in &gh.teams {
org_team_members
.entry(toml_gh_team.org.clone())
.or_default()
.extend(toml_gh_team.members.iter().copied());
}
}
}
org_team_members
}

/// Diff organization memberships between TOML teams and GitHub
fn diff_org_memberships(&self) -> anyhow::Result<Vec<OrgMembershipDiff>> {
let toml_org_team_members = self.get_org_members_from_teams();

let mut org_diffs: BTreeMap<String, OrgMembershipDiff> = BTreeMap::new();

for (org, toml_members) in toml_org_team_members {
let Some(gh_org_members) = self.org_members.get(&org) else {
panic!("GitHub organization {org} not found");
};

let members_to_remove = self.members_to_remove(toml_members, gh_org_members);

// The rest are members that should be removed
if !members_to_remove.is_empty() {
let mut members_to_remove: Vec<String> = members_to_remove.into_values().collect();
members_to_remove.sort();

org_diffs.insert(
org.clone(),
OrgMembershipDiff {
org,
members_to_remove,
},
);
}
}

Ok(org_diffs.into_values().collect())
}

/// Return GitHub members that should be removed from the organization.
fn members_to_remove(
&self,
toml_members: HashSet<u64>,
gh_org_members: &HashMap<u64, String>,
) -> HashMap<u64, String> {
// Initialize `members_to_remove` to all GitHub members in the org.
// Next, we'll delete members from `members_to_remove` that don't respect certain criteria.
let mut members_to_remove = gh_org_members.clone();

// People who belong to a team should stay in the org.
for member in toml_members {
members_to_remove.remove(&member);
}

// Members that are explicitly allowed in the `config.toml` file should stay in the org.
for allowed_member in &self.config.allowed_org_members {
if let Some(member_to_retain) = members_to_remove
.iter()
.find(|(_, username)| username == &allowed_member)
.map(|(id, _)| *id)
{
members_to_remove.remove(&member_to_retain);
}
}
members_to_remove
}

fn diff_teams(&self) -> anyhow::Result<Vec<TeamDiff>> {
let mut diffs = Vec::new();
let mut unseen_github_teams = HashMap::new();
Expand Down Expand Up @@ -584,6 +670,7 @@ const BOTS_TEAMS: &[&str] = &["bors", "highfive", "rfcbot", "bots"];
pub(crate) struct Diff {
team_diffs: Vec<TeamDiff>,
repo_diffs: Vec<RepoDiff>,
org_membership_diffs: Vec<OrgMembershipDiff>,
}

impl Diff {
Expand All @@ -595,12 +682,17 @@ impl Diff {
for repo_diff in self.repo_diffs {
repo_diff.apply(sync)?;
}
for org_diff in self.org_membership_diffs {
org_diff.apply(sync)?;
}

Ok(())
}

pub(crate) fn is_empty(&self) -> bool {
self.team_diffs.is_empty() && self.repo_diffs.is_empty()
self.team_diffs.is_empty()
&& self.repo_diffs.is_empty()
&& self.org_membership_diffs.is_empty()
}
}

Expand All @@ -620,6 +712,13 @@ impl std::fmt::Display for Diff {
}
}

if !&self.org_membership_diffs.is_empty() {
writeln!(f, "💻 Org membership Diffs:")?;
for org_diff in &self.org_membership_diffs {
write!(f, "{org_diff}")?;
}
}

Ok(())
}
}
Expand Down Expand Up @@ -655,6 +754,34 @@ impl std::fmt::Display for RepoDiff {
}
}

#[derive(Debug)]
struct OrgMembershipDiff {
org: OrgName,
members_to_remove: Vec<String>,
}

impl OrgMembershipDiff {
fn apply(self, sync: &GitHubWrite) -> anyhow::Result<()> {
for member in &self.members_to_remove {
sync.remove_gh_member_from_org(&self.org, member)?;
}

Ok(())
}
}

impl std::fmt::Display for OrgMembershipDiff {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
if !self.members_to_remove.is_empty() {
writeln!(f, "❌ Removing the following members from `{}`:", self.org)?;
for member in &self.members_to_remove {
writeln!(f, " - {member}",)?;
}
}
Ok(())
}
}

#[derive(Debug)]
struct CreateRepoDiff {
org: String,
Expand Down
Loading