Don't use the same API key for other organizations we are pulling from

```EOF
For the organizations list, I am trying use my test instance, but getting the following in the logs;
```
2025-09-23T00:12:38.638052Z  INFO gitea_mirror: Fetching repositories from organization: https://gitea.hak8or.com/mirrors
2025-09-23T00:12:38.638081Z  INFO fetch_org_repos{org_url="https://gitea.hak8or.com/mirrors"}: gitea_mirror: Querying API endpoint: https://gitea.hak8or.com/api/v1/users/mirrors/repos
2025-09-23T00:12:38.653694Z ERROR gitea_mirror: Failed to fetch repos from https://gitea.hak8or.com/mirrors: HTTP status client error (401 Unauthorized) for url (https://gitea.hak8or.com/api/v1/users/mirrors/repos?page=1)
2025-09-23T00:12:38.653713Z  INFO gitea_mirror: Gitea mirror process completed.
```

I don't have a user with that key for the instance. Can you add the ability to provide an api key to each organization entry in the toml config? At the same time, is it possible to get a list of all repos from an organization without needing to use an api key? If so, when no api key is provided, can you use that?
```EOF
This commit is contained in:
2025-09-22 20:29:14 -04:00
parent f732535db2
commit 129d67bc8b

View File

@@ -31,13 +31,20 @@ struct RepoConfig {
rename: Option<String>, rename: Option<String>,
} }
// Represents a single organization entry in the config file.
#[derive(Deserialize, Debug, Clone)]
struct OrgConfig {
url: String,
api_key: Option<String>,
}
// Represents the main structure of the TOML configuration file. // Represents the main structure of the TOML configuration file.
#[derive(Deserialize, Debug)] #[derive(Deserialize, Debug)]
struct Config { struct Config {
gitea_url: String, gitea_url: String,
api_key: String, api_key: String,
repos: Option<Vec<RepoConfig>>, repos: Option<Vec<RepoConfig>>,
organizations: Option<Vec<String>>, organizations: Option<Vec<OrgConfig>>,
} }
// Represents the payload for creating a migration in Gitea. // Represents the payload for creating a migration in Gitea.
@@ -51,6 +58,12 @@ struct MigrateRepoPayload<'a> {
uid: i64, // The user ID of the owner. We'll fetch this. uid: i64, // The user ID of the owner. We'll fetch this.
} }
// Represents a repository as returned by the Gitea API.
#[derive(Deserialize, Debug)]
struct GiteaRepo {
name: String,
}
// Represents a user as returned by the Gitea API. // Represents a user as returned by the Gitea API.
#[derive(Deserialize, Debug)] #[derive(Deserialize, Debug)]
struct GiteaUser { struct GiteaUser {
@@ -113,12 +126,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
} }
// Process repositories from the organizations/users list. // Process repositories from the organizations/users list.
if let Some(org_urls) = &config.organizations { if let Some(org_configs) = &config.organizations {
for org_url in org_urls { for org_config in org_configs {
info!("Fetching repositories from organization: {}", org_url); info!(
match fetch_org_repos(&http_client, org_url, &config.api_key).await { "Fetching repositories from organization: {}",
org_config.url
);
match fetch_org_repos(&http_client, &org_config.url, org_config.api_key.as_deref())
.await
{
Ok(repo_urls) => { Ok(repo_urls) => {
info!("Found {} repositories for {}", repo_urls.len(), org_url); info!(
"Found {} repositories for {}",
repo_urls.len(),
org_config.url
);
for url in repo_urls { for url in repo_urls {
process_repo( process_repo(
&url, &url,
@@ -131,7 +153,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.await?; .await?;
} }
} }
Err(e) => error!("Failed to fetch repos from {}: {}", org_url, e), Err(e) => error!("Failed to fetch repos from {}: {}", org_config.url, e),
} }
} }
} }
@@ -223,7 +245,7 @@ async fn create_migration(
async fn fetch_org_repos( async fn fetch_org_repos(
http_client: &reqwest::Client, http_client: &reqwest::Client,
org_url: &str, org_url: &str,
api_key: &str, api_key: Option<&str>,
) -> Result<Vec<String>, Box<dyn std::error::Error>> { ) -> Result<Vec<String>, Box<dyn std::error::Error>> {
// This is a simplified fetcher. It assumes Gitea API compatibility. // This is a simplified fetcher. It assumes Gitea API compatibility.
// For GitHub, you might need a different base URL and auth method. // For GitHub, you might need a different base URL and auth method.
@@ -247,12 +269,17 @@ async fn fetch_org_repos(
let mut repos: Vec<String> = Vec::new(); let mut repos: Vec<String> = Vec::new();
let mut page = 1; let mut page = 1;
loop { loop {
let response: Vec<serde_json::Value> = http_client let mut request_builder = http_client
.get(&api_url) .get(&api_url)
.query(&[("page", page.to_string())]) .query(&[("page", page.to_string())])
// For GitHub, a User-Agent is required. // For GitHub, a User-Agent is required.
.header("User-Agent", "gitea-mirror-rust-client") .header("User-Agent", "gitea-mirror-rust-client");
.header("Authorization", format!("token {}", api_key))
if let Some(key) = api_key {
request_builder = request_builder.header("Authorization", format!("token {}", key));
}
let response: Vec<serde_json::Value> = request_builder
.send() .send()
.await? .await?
.error_for_status()? .error_for_status()?