diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5786cf6..8bae31d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: check - args: --no-default-features --features=secure-auth,${{ matrix.db_feature }},crates-io-mirroring + args: --no-default-features --features=${{ matrix.db_feature }},crates-io-mirroring check_openid: name: Check ${{ matrix.db_feature }} with OpenId (${{ matrix.os }}) @@ -59,7 +59,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: check - args: --no-default-features --features=secure-auth,${{ matrix.db_feature }},crates-io-mirroring,openid + args: --no-default-features --features=${{ matrix.db_feature }},crates-io-mirroring,openid test: @@ -86,7 +86,7 @@ jobs: uses: actions-rs/cargo@v1 with: command: test - args: --no-default-features --features=secure-auth,${{ matrix.db_feature }},crates-io-mirroring + args: --no-default-features --features=${{ matrix.db_feature }},crates-io-mirroring lints: name: Lints (${{ matrix.os }}) diff --git a/Cargo.toml b/Cargo.toml index 545c159..594a746 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,56 +1,69 @@ [package] name = "ktra" + version = "0.7.0" -authors = ["moriturus "] +authors = [ "moriturus " ] +description = "Your Little Cargo Registry" +documentation = "https://book.ktra.dev" edition = "2018" license = "MIT OR Apache-2.0" -description = "Your Little Cargo Registry" repository = "https://github.com/moriturus/ktra" -keywords = ["ktra", "registry", "cargo"] -categories = ["command-line-utilities", "development-tools"] +keywords = [ "ktra", "registry", "cargo" ] +categories = [ "command-line-utilities", "development-tools" ] readme = "README.md" -documentation = "https://book.ktra.dev" [features] -default = ["secure-auth", "db-sled", "crates-io-mirroring"] +default = [ "db-sled", "crates-io-mirroring", "cli" ] -secure-auth = ["rand", "rust-argon2"] -crates-io-mirroring = ["reqwest", "tokio-util"] -mirroring-dummy = [] -openid = ["openidconnect", "reqwest"] -db-sled = ["sled"] -db-redis = ["redis"] -db-mongo = ["mongodb", "bson"] +crates-io-mirroring = [ "reqwest" ] +openid = [ "openidconnect", "reqwest" ] +db-sled = [ "sled" ] +db-redis = [ "redis" ] +db-mongo = [ "mongodb", "bson" ] +cli = [ "tracing-subscriber", "clap", "toml", "anyhow" ] [dependencies] -tokio = { version = "1.1", features = ["macros", "rt-multi-thread", "fs", "io-util"] } +tokio = { version = "1.1", features = [ + "macros", + "rt-multi-thread", + "fs", + "io-util", +] } warp = "0.3" -serde = { version = "1.0", features = ["derive"] } +serde = { version = "1.0", features = [ "derive" ] } serde_json = "1.0" tracing = "0.1" -tracing-subscriber = "0.2" -tracing-futures = "0.2" futures = "0.3" -semver = { version = "0.11", features = ["serde"] } -url = { version = "2.2", features = ["serde"] } -anyhow = "1.0" +semver = { version = "0.11", features = [ "serde" ] } +url = { version = "2.2", features = [ "serde" ] } thiserror = "1.0" git2 = "0.13" bytes = "1.0" sha2 = "0.9" -toml = "0.5" -clap = "2.33" async-trait = "0.1" -reqwest = { version = "0.11", features = ["gzip", "brotli", "json"], optional = true } -tokio-util = { version = "0.6", features = ["io"], optional = true } +reqwest = { version = "0.11", features = [ + "gzip", + "brotli", + "json", +], optional = true } -rand = { version = "0.8", optional = true } -rust-argon2 = { version = "0.8", optional = true } +rand = "0.8" +rust-argon2 = "0.8" sled = { version = "0.34", optional = true } -redis = { version = "0.19", features = ["tokio-comp"], optional = true } +redis = { version = "0.19", features = [ "tokio-comp" ], optional = true } mongodb = { version = "1.1", optional = true } -bson = { version = "1.1", features = ["u2i"], optional = true } +bson = { version = "1.1", features = [ "u2i" ], optional = true } openidconnect = { version = "2.1.1", optional = true } + +tracing-subscriber = { version = "0.2", optional = true } +clap = { version = "2.33", optional = true } +toml = { version = "0.5", optional = true } +anyhow = { version = "1.0", optional = true } + +[[bin]] +name = "ktra" +path = "bin/ktra.rs" +required-features = [ "cli" ] diff --git a/src/main.rs b/bin/ktra.rs similarity index 65% rename from src/main.rs rename to bin/ktra.rs index c9a579e..a5fc217 100644 --- a/src/main.rs +++ b/bin/ktra.rs @@ -1,108 +1,18 @@ #![type_length_limit = "2000000"] -mod config; -mod db_manager; -mod delete; -mod error; -mod get; -mod index_manager; -mod models; -mod openid; -mod post; -mod put; -mod utils; - -use crate::config::Config; -use crate::index_manager::IndexManager; -use clap::{clap_app, crate_authors, crate_version, ArgMatches}; -use db_manager::DbManager; -#[cfg(feature = "crates-io-mirroring")] -use reqwest::Client; -use std::convert::Infallible; use std::path::{Path, PathBuf}; use std::sync::Arc; -use tokio::sync::RwLock; -use warp::{Filter, Rejection, Reply}; - -#[cfg(all( - feature = "db-mongo", - not(all(feature = "db-redis", feature = "db-sled")) -))] -use db_manager::MongoDbManager; -#[cfg(all( - feature = "db-redis", - not(all(feature = "db-sled", feature = "db-mongo")) -))] -use db_manager::RedisDbManager; -#[cfg(all( - feature = "db-sled", - not(all(feature = "db-redis", feature = "db-mongo")) -))] -use db_manager::SledDbManager; - -#[cfg(feature = "crates-io-mirroring")] -#[tracing::instrument(skip( - db_manager, - index_manager, - dl_dir_path, - http_client, - cache_dir_path, - dl_path -))] -fn apis( - db_manager: Arc>, - index_manager: Arc, - dl_dir_path: Arc, - http_client: Client, - cache_dir_path: Arc, - dl_path: Vec, -) -> impl Filter + Clone { - let routes = get::apis( - db_manager.clone(), - dl_dir_path.clone(), - http_client, - cache_dir_path, - dl_path, - ) - .or(delete::apis(db_manager.clone(), index_manager.clone())) - .or(put::apis(db_manager.clone(), index_manager, dl_dir_path)); - #[cfg(not(feature = "openid"))] - let routes = routes.or(post::apis(db_manager.clone())); - routes -} -#[cfg(not(feature = "crates-io-mirroring"))] -#[tracing::instrument(skip(db_manager, index_manager, dl_dir_path, dl_path))] -fn apis( - db_manager: Arc>, - index_manager: Arc, - dl_dir_path: Arc, - dl_path: Vec, -) -> impl Filter + Clone { - let routes = get::apis(db_manager.clone(), dl_dir_path.clone(), dl_path) - .or(delete::apis(db_manager.clone(), index_manager.clone())) - .or(put::apis(db_manager, index_manager, dl_dir_path)); - #[cfg(not(feature = "openid"))] - let routes = routes.or(post::apis(db_manager.clone())); - routes -} +use clap::{clap_app, crate_authors, crate_version, ArgMatches}; +use futures::TryFutureExt; +use serde::Deserialize; +use tokio::fs::OpenOptions; +use tokio::io::{AsyncReadExt, BufReader}; +use tokio::sync::RwLock; +use warp::Filter; -#[tracing::instrument(skip(rejection))] -async fn handle_rejection(rejection: Rejection) -> Result { - if let Some(application_error) = rejection.find::() { - let (json, status_code) = application_error.to_reply(); - Ok(warp::reply::with_status(json, status_code)) - } else { - Ok(warp::reply::with_status( - warp::reply::json(&serde_json::json!({ - "errors": [ - { "detail": "resource or api is not defined" } - ] - })), - warp::http::StatusCode::NOT_FOUND, - )) - } -} +use ktra::apis; +use ktra::config::{CrateFilesConfig, IndexConfig, OpenIdConfig, ServerConfig}; #[tracing::instrument(skip(config))] async fn run_server(config: Config) -> anyhow::Result<()> { @@ -112,11 +22,7 @@ async fn run_server(config: Config) -> anyhow::Result<()> { ); tokio::fs::create_dir_all(&config.crate_files_config.dl_dir_path).await?; - #[cfg(feature = "crates-io-mirroring")] - tokio::fs::create_dir_all(&config.crate_files_config.cache_dir_path).await?; let dl_dir_path = config.crate_files_config.dl_dir_path.clone(); - #[cfg(feature = "crates-io-mirroring")] - let cache_dir_path = config.crate_files_config.cache_dir_path.clone(); let dl_path = config.crate_files_config.dl_path.clone(); let server_config = config.server_config.clone(); @@ -124,44 +30,63 @@ async fn run_server(config: Config) -> anyhow::Result<()> { feature = "db-sled", not(all(feature = "db-redis", feature = "db-mongo")) ))] - let db_manager = SledDbManager::new(&config.db_config).await?; + let db_manager = ktra::db_manager::SledDbManager::new( + config.db_config.db_dir_path, + config.db_config.login_prefix, + ) + .await?; #[cfg(all( feature = "db-redis", not(all(feature = "db-sled", feature = "db-mongo")) ))] - let db_manager = RedisDbManager::new(&config.db_config).await?; + let db_manager = ktra::db_manager::RedisDbManager::new( + config.db_config.redis_url, + config.db_config.login_prefix, + ) + .await?; #[cfg(all( feature = "db-mongo", not(all(feature = "db-sled", feature = "db-redis")) ))] - let db_manager = MongoDbManager::new(&config.db_config).await?; - let index_manager = IndexManager::new(config.index_config).await?; - index_manager.pull().await?; + let db_manager = ktra::db_manager::MongoDbManager::new( + config.db_config.mongodb_url, + config.db_config.login_prefix, + ) + .await?; - #[cfg(feature = "crates-io-mirroring")] - let http_client = Client::builder().build()?; + let index_manager = ktra::IndexManager::new(config.index_config).await?; + index_manager.pull().await?; let db_manager = Arc::new(RwLock::new(db_manager)); - let routes = apis( + let routes = apis::registry::apis( db_manager.clone(), Arc::new(index_manager), Arc::new(dl_dir_path), - #[cfg(feature = "crates-io-mirroring")] - http_client, - #[cfg(feature = "crates-io-mirroring")] - Arc::new(cache_dir_path), dl_path, ); + #[cfg(feature = "crates-io-mirroring")] + let routes = { + tokio::fs::create_dir_all(&config.crate_files_config.cache_dir_path).await?; + let cache_dir_path = config.crate_files_config.cache_dir_path.clone(); + routes.or(apis::mirroring::download_crates_io( + reqwest::Client::builder().build()?, + Arc::new(cache_dir_path), + )) + }; + #[cfg(feature = "openid")] - let routes = routes.or(openid::apis( + let routes = routes.or(apis::openid::apis( db_manager.clone(), Arc::new(config.openid_config), )); + #[cfg(not(feature = "openid"))] + let routes = routes.or(apis::user::apis(db_manager.clone())); + let routes = routes .with(warp::trace::request()) - .recover(handle_rejection); + .recover(ktra::utils::handle_rejection); warp::serve(routes) .run(server_config.to_socket_addr()) @@ -172,13 +97,100 @@ async fn run_server(config: Config) -> anyhow::Result<()> { #[tracing::instrument(skip(path))] async fn config(path: impl AsRef) -> anyhow::Result { let path = path.as_ref(); + if path.exists() { - Config::open(path).await + let mut file = OpenOptions::new() + .read(true) + .open(path) + .map_ok(BufReader::new) + .await?; + let mut buf = String::new(); + file.read_to_string(&mut buf).await?; + + toml::from_str(&buf).map_err(Into::into) } else { Ok(Config::default()) } } +#[derive(Debug, Clone, Deserialize)] +pub struct DbConfig { + #[serde(default = "DbConfig::login_prefix_default")] + pub login_prefix: String, + + #[cfg(feature = "db-sled")] + #[serde(default = "DbConfig::db_dir_path_default")] + pub db_dir_path: PathBuf, + + #[cfg(feature = "db-redis")] + #[serde(default = "DbConfig::redis_url_default")] + pub redis_url: String, + + #[cfg(feature = "db-mongo")] + #[serde(default = "DbConfig::mongodb_url_default")] + pub mongodb_url: String, +} + +impl Default for DbConfig { + fn default() -> DbConfig { + DbConfig { + login_prefix: DbConfig::login_prefix_default(), + #[cfg(feature = "db-sled")] + db_dir_path: DbConfig::db_dir_path_default(), + #[cfg(feature = "db-redis")] + redis_url: DbConfig::redis_url_default(), + #[cfg(feature = "db-mongo")] + mongodb_url: DbConfig::mongodb_url_default(), + } + } +} + +impl DbConfig { + fn login_prefix_default() -> String { + "ktra-secure-auth:".to_owned() + } + + #[cfg(feature = "db-sled")] + fn db_dir_path_default() -> PathBuf { + PathBuf::from("db") + } + + #[cfg(feature = "db-redis")] + fn redis_url_default() -> String { + "redis://localhost".to_owned() + } + + #[cfg(feature = "db-mongo")] + fn mongodb_url_default() -> String { + "mongodb://localhost:27017".to_owned() + } +} +#[derive(Debug, Clone, Deserialize)] +pub struct Config { + #[serde(default)] + pub crate_files_config: CrateFilesConfig, + #[serde(default)] + pub db_config: DbConfig, + #[serde(default)] + pub index_config: IndexConfig, + #[serde(default)] + pub server_config: ServerConfig, + #[serde(default)] + pub openid_config: OpenIdConfig, +} + +impl Default for Config { + fn default() -> Config { + Config { + crate_files_config: Default::default(), + db_config: Default::default(), + index_config: Default::default(), + server_config: Default::default(), + openid_config: Default::default(), + } + } +} + #[tracing::instrument] fn matches() -> ArgMatches<'static> { clap_app!(ktra => @@ -320,41 +332,37 @@ async fn main() -> anyhow::Result<()> { } #[cfg(feature = "openid")] - if let Some(issuer) = matches.value_of("OPENID_ISSUER").map(ToOwned::to_owned) { - config.openid_config.issuer_url = issuer; - } - - #[cfg(feature = "openid")] - if let Some(redirect) = matches.value_of("OPENID_REDIRECT").map(ToOwned::to_owned) { - config.openid_config.redirect_url = redirect; - } - - #[cfg(feature = "openid")] - if let Some(client_id) = matches.value_of("OPENID_APP_ID").map(ToOwned::to_owned) { - config.openid_config.client_id = client_id; - } - - #[cfg(feature = "openid")] - if let Some(client_secret) = matches.value_of("OPENID_APP_SECRET").map(ToOwned::to_owned) { - config.openid_config.client_secret = client_secret; - } - - #[cfg(feature = "openid")] - if let Some(scopes) = matches.value_of("OPENID_ADD_SCOPES").map(ToOwned::to_owned) { - config.openid_config.additional_scopes = - scopes.split(',').map(ToString::to_string).collect(); - } - - #[cfg(feature = "openid")] - if let Some(gitlab_groups) = matches.value_of("OPENID_GITLAB_GROUPS") { - config.openid_config.gitlab_authorized_groups = - Some(gitlab_groups.split(',').map(ToString::to_string).collect()); - } - - #[cfg(feature = "openid")] - if let Some(gitlab_users) = matches.value_of("OPENID_GITLAB_USERS") { - config.openid_config.gitlab_authorized_users = - Some(gitlab_users.split(',').map(ToString::to_string).collect()); + { + if let Some(issuer) = matches.value_of("OPENID_ISSUER").map(ToOwned::to_owned) { + config.openid_config.issuer_url = issuer; + } + + if let Some(redirect) = matches.value_of("OPENID_REDIRECT").map(ToOwned::to_owned) { + config.openid_config.redirect_url = redirect; + } + + if let Some(client_id) = matches.value_of("OPENID_APP_ID").map(ToOwned::to_owned) { + config.openid_config.client_id = client_id; + } + + if let Some(client_secret) = matches.value_of("OPENID_APP_SECRET").map(ToOwned::to_owned) { + config.openid_config.client_secret = client_secret; + } + + if let Some(scopes) = matches.value_of("OPENID_ADD_SCOPES").map(ToOwned::to_owned) { + config.openid_config.additional_scopes = + scopes.split(',').map(ToString::to_string).collect(); + } + + if let Some(gitlab_groups) = matches.value_of("OPENID_GITLAB_GROUPS") { + config.openid_config.gitlab_authorized_groups = + Some(gitlab_groups.split(',').map(ToString::to_string).collect()); + } + + if let Some(gitlab_users) = matches.value_of("OPENID_GITLAB_USERS") { + config.openid_config.gitlab_authorized_users = + Some(gitlab_users.split(',').map(ToString::to_string).collect()); + } } run_server(config).await diff --git a/docker/ktra.Dockerfile b/docker/ktra.Dockerfile index 1456d6a..13805f6 100644 --- a/docker/ktra.Dockerfile +++ b/docker/ktra.Dockerfile @@ -15,7 +15,7 @@ COPY --chown=rust:rust ./src /build/src COPY --chown=rust:rust ./Cargo.toml /build/ WORKDIR /build -RUN cargo build --release --no-default-features --features=secure-auth,${DB},${MIRRORING} +RUN cargo build --release --no-default-features --features=${DB},${MIRRORING} FROM debian:bullseye-slim diff --git a/docker/ktra_openid.Dockerfile b/docker/ktra_openid.Dockerfile index ed10613..2b23602 100644 --- a/docker/ktra_openid.Dockerfile +++ b/docker/ktra_openid.Dockerfile @@ -15,7 +15,7 @@ COPY --chown=rust:rust ./src /build/src COPY --chown=rust:rust ./Cargo.toml /build/ WORKDIR /build -RUN cargo build --release --no-default-features --features=secure-auth,openid,${DB},${MIRRORING} +RUN cargo build --release --no-default-features --features=openid,${DB},${MIRRORING} FROM debian:bullseye-slim diff --git a/src/apis/mirroring.rs b/src/apis/mirroring.rs new file mode 100644 index 0000000..9eff505 --- /dev/null +++ b/src/apis/mirroring.rs @@ -0,0 +1,114 @@ +#![cfg(feature = "crates-io-mirroring")] + +use std::path::PathBuf; +use std::sync::Arc; + +use futures::TryFutureExt; +use reqwest::Client; +use semver::Version; +use tokio::fs::OpenOptions; +use tokio::io::AsyncReadExt; +use tokio::io::{AsyncWriteExt, BufWriter}; +use url::Url; +use warp::http::Response; +use warp::hyper::body::Bytes; +use warp::{Filter, Rejection, Reply}; + +use crate::error::Error; +use crate::utils::{file_exists_and_not_empty, with_cache_dir_path, with_http_client}; + +#[tracing::instrument(skip(http_client, cache_dir_path, crate_name, version))] +async fn cache_crate_file( + http_client: Client, + cache_dir_path: Arc, + crate_name: impl AsRef, + version: Version, +) -> Result { + let computation = async move { + let mut cache_dir_path = cache_dir_path.as_ref().to_path_buf(); + let crate_components = format!("{}/{}/download", crate_name.as_ref(), version); + cache_dir_path.push(&crate_components); + let cache_file_path = cache_dir_path; + + if file_exists_and_not_empty(&cache_file_path).await { + OpenOptions::new() + .write(false) + .create(false) + .read(true) + .open(cache_file_path) + .and_then(|mut file| async move { + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).await?; + Ok(Bytes::from(buffer)) + }) + .map_err(Error::Io) + .await + } else { + let mut crate_dir_path = cache_file_path.clone(); + crate_dir_path.pop(); + let crate_dir_path = crate_dir_path; + + tokio::fs::create_dir_all(crate_dir_path) + .map_err(Error::Io) + .await?; + + let file = OpenOptions::new() + .write(true) + .create(true) + .read(true) + .open(&cache_file_path) + .map_err(Error::Io) + .await?; + let mut file = BufWriter::with_capacity(128 * 1024, file); + + let crates_io_base_url = + Url::parse("https://crates.io/api/v1/crates/").map_err(Error::UrlParsing)?; + let crate_file_url = crates_io_base_url + .join(&crate_components) + .map_err(Error::UrlParsing)?; + let body = http_client + .get(crate_file_url) + .send() + .and_then(|res| async move { res.error_for_status() }) + .and_then(|res| res.bytes()) + .map_err(Error::HttpRequest) + .await?; + + if body.is_empty() { + return Err(Error::InvalidHttpResponseLength); + } + + file.write_all(&body).map_err(Error::Io).await?; + file.flush().map_err(Error::Io).await?; + + Ok(body) + } + }; + + computation.map_err(warp::reject::custom).await +} + +#[tracing::instrument(skip(cache_dir_path))] +pub fn download_crates_io( + http_client: Client, + cache_dir_path: Arc, +) -> impl Filter + Clone { + warp::get() + .and(with_http_client(http_client)) + .and(with_cache_dir_path(cache_dir_path)) + .and(warp::path!( + "ktra" / "api" / "v1" / "mirror" / String / Version / "download" + )) + .and_then(cache_crate_file) + .and_then(handle_download_crates_io) +} + +#[tracing::instrument(skip(crate_file_data))] +async fn handle_download_crates_io(crate_file_data: Bytes) -> Result { + let response = Response::builder() + .header("Content-Type", "application/x-tar") + .body(crate_file_data) + .map_err(Error::HttpResponseBuilding)?; + + Ok(response) +} diff --git a/src/apis/mod.rs b/src/apis/mod.rs new file mode 100644 index 0000000..25174d0 --- /dev/null +++ b/src/apis/mod.rs @@ -0,0 +1,5 @@ +pub mod mirroring; +#[cfg(feature = "openid")] +pub mod openid; +pub mod registry; +pub mod user; diff --git a/src/openid.rs b/src/apis/openid.rs similarity index 95% rename from src/openid.rs rename to src/apis/openid.rs index 78a5fd5..9b704ba 100644 --- a/src/openid.rs +++ b/src/apis/openid.rs @@ -3,7 +3,7 @@ use crate::config::OpenIdConfig; use crate::db_manager::DbManager; use crate::error::Error; -use crate::models::{Claims, CodeQuery, User}; +use crate::models::User; use crate::utils::*; use futures::TryFutureExt; use openidconnect::core::{ @@ -14,10 +14,28 @@ use openidconnect::{ AdditionalClaims, AuthenticationFlow, AuthorizationCode, ClientId, ClientSecret, CsrfToken, IssuerUrl, Nonce, OAuth2TokenResponse, RedirectUrl, Scope, UserInfoClaims, }; +use serde::{Deserialize, Serialize}; use std::sync::Arc; use tokio::sync::RwLock; use warp::{Filter, Rejection, Reply}; +#[derive(Debug, Clone, Deserialize)] +pub struct CodeQuery { + pub code: String, + pub state: Option, +} + +/// The additional claims OpenId providers may send +/// +/// All fields here are options so that the extra claims are caught when presents +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Claims { + pub(crate) sub: Option, + pub(crate) sub_legacy: Option, + // Gitlab claims return the groups a user is in. + // This property is used when gitlab_authorized_groups is set in the configuration + pub(crate) groups: Option>, +} impl AdditionalClaims for Claims {} #[tracing::instrument(skip(db_manager, openid_config))] diff --git a/src/delete.rs b/src/apis/registry/delete.rs similarity index 100% rename from src/delete.rs rename to src/apis/registry/delete.rs diff --git a/src/apis/registry/get.rs b/src/apis/registry/get.rs new file mode 100644 index 0000000..685daa8 --- /dev/null +++ b/src/apis/registry/get.rs @@ -0,0 +1,93 @@ +use crate::db_manager::DbManager; +use crate::models::{Query, User}; +use crate::utils::*; +use futures::TryFutureExt; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::RwLock; +use warp::{filters::BoxedFilter, Filter, Rejection, Reply}; + +#[tracing::instrument(skip(db_manager, dl_dir_path, path))] +pub fn apis( + db_manager: Arc>, + dl_dir_path: Arc, + path: Vec, +) -> impl Filter + Clone { + let routes = download(dl_dir_path, path) + .or(owners(db_manager.clone())) + .or(search(db_manager)); + + routes +} + +#[tracing::instrument(skip(path))] +pub(crate) fn into_boxed_filters(path: Vec) -> BoxedFilter<()> { + let (h, t) = path.split_at(1); + t.iter().fold(warp::path(h[0].clone()).boxed(), |accm, s| { + accm.and(warp::path(s.clone())).boxed() + }) +} + +#[tracing::instrument(skip(path, dl_dir_path))] +fn download( + dl_dir_path: Arc, + path: Vec, +) -> impl Filter + Clone { + into_boxed_filters(path).and(warp::fs::dir(dl_dir_path.to_path_buf())) +} + +#[tracing::instrument(skip(db_manager))] +fn owners( + db_manager: Arc>, +) -> impl Filter + Clone { + warp::get() + .and(with_db_manager(db_manager)) + .and(authorization_header()) + .and(warp::path!("api" / "v1" / "crates" / String / "owners")) + .and_then(handle_owners) +} + +#[tracing::instrument(skip(db_manager, _token, name))] +async fn handle_owners( + db_manager: Arc>, + // `token` is not a used argument. + // the specification demands that the authorization is required but listing owners api does not update the database. + _token: String, + name: String, +) -> Result { + let db_manager = db_manager.read().await; + let owners = db_manager + .owners(&name) + .map_err(warp::reject::custom) + .await?; + Ok(owners_json(owners)) +} + +#[tracing::instrument(skip(db_manager))] +fn search( + db_manager: Arc>, +) -> impl Filter + Clone { + warp::get() + .and(with_db_manager(db_manager)) + .and(warp::path!("api" / "v1" / "crates")) + .and(warp::query::()) + .and_then(handle_search) +} + +#[tracing::instrument(skip(db_manager, query))] +async fn handle_search( + db_manager: Arc>, + query: Query, +) -> Result { + let db_manager = db_manager.read().await; + db_manager + .search(&query) + .map_ok(|s| warp::reply::json(&s)) + .map_err(warp::reject::custom) + .await +} + +#[tracing::instrument(skip(owners))] +fn owners_json(owners: Vec) -> impl Reply { + warp::reply::json(&serde_json::json!({ "users": owners })) +} diff --git a/src/apis/registry/mod.rs b/src/apis/registry/mod.rs new file mode 100644 index 0000000..9690c35 --- /dev/null +++ b/src/apis/registry/mod.rs @@ -0,0 +1,30 @@ +use std::{convert::Infallible, path::PathBuf, sync::Arc}; + +use tokio::sync::RwLock; +use warp::{Filter, Rejection, Reply}; + +use crate::{db_manager::DbManager, index_manager::IndexManager}; + +pub mod delete; +pub mod get; +pub mod put; + +#[tracing::instrument(skip(db_manager, index_manager, dl_dir_path, dl_path))] +pub fn apis( + db_manager: Arc>, + index_manager: Arc, + dl_dir_path: Arc, + dl_path: Vec, +) -> impl Filter + Clone { + let routes = self::get::apis(db_manager.clone(), dl_dir_path.clone(), dl_path) + .or(self::delete::apis( + db_manager.clone(), + index_manager.clone(), + )) + .or(self::put::apis( + db_manager.clone(), + index_manager, + dl_dir_path, + )); + routes +} diff --git a/src/put.rs b/src/apis/registry/put.rs similarity index 100% rename from src/put.rs rename to src/apis/registry/put.rs diff --git a/src/post.rs b/src/apis/user.rs similarity index 93% rename from src/post.rs rename to src/apis/user.rs index 8e2125a..05824af 100644 --- a/src/post.rs +++ b/src/apis/user.rs @@ -18,6 +18,7 @@ pub fn apis( new_user(db_manager.clone()) .or(login(db_manager.clone())) .or(change_password(db_manager)) + .or(me()) } #[tracing::instrument(skip(db_manager))] @@ -157,3 +158,10 @@ async fn handle_change_password( Err(Error::InvalidPassword).map_err(warp::reject::custom) } } + +#[tracing::instrument] +fn me() -> impl Filter + Clone { + warp::get() + .and(warp::path!("me")) + .map(|| "$ curl -X POST -H 'Content-Type: application/json' -d '{\"password\":\"YOUR PASSWORD\"}' https:///ktra/api/v1/login/") +} diff --git a/src/config.rs b/src/config.rs index 088fc4f..7eb97f7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,11 +1,8 @@ -use futures::TryFutureExt; use serde::Deserialize; use std::net::SocketAddr; -use std::path::{Path, PathBuf}; -use tokio::fs::OpenOptions; -use tokio::io::{AsyncReadExt, BufReader}; +use std::path::PathBuf; -#[derive(Debug, Clone, Deserialize, Default)] +#[derive(Debug, Clone, Deserialize)] pub struct IndexConfig { pub remote_url: String, #[serde(default = "IndexConfig::local_path_default")] @@ -23,6 +20,24 @@ pub struct IndexConfig { pub email: Option, } +impl Default for IndexConfig { + fn default() -> Self { + Self { + remote_url: Default::default(), + local_path: Self::local_path_default(), + branch: Self::branch_default(), + https_username: Default::default(), + https_password: Default::default(), + ssh_username: Default::default(), + ssh_pubkey_path: Default::default(), + ssh_privkey_path: Default::default(), + ssh_key_passphrase: Default::default(), + name: Self::name_default(), + email: Default::default(), + } + } +} + impl IndexConfig { fn local_path_default() -> PathBuf { PathBuf::from("index") @@ -74,59 +89,6 @@ impl CrateFilesConfig { } } -#[derive(Debug, Clone, Deserialize)] -pub struct DbConfig { - #[serde(default = "DbConfig::login_prefix_default")] - pub login_prefix: String, - - #[cfg(feature = "db-sled")] - #[serde(default = "DbConfig::db_dir_path_default")] - pub db_dir_path: PathBuf, - - #[cfg(feature = "db-redis")] - #[serde(default = "DbConfig::redis_url_default")] - pub redis_url: String, - - #[cfg(feature = "db-mongo")] - #[serde(default = "DbConfig::mongodb_url_default")] - pub mongodb_url: String, -} - -impl Default for DbConfig { - fn default() -> DbConfig { - DbConfig { - login_prefix: DbConfig::login_prefix_default(), - #[cfg(feature = "db-sled")] - db_dir_path: DbConfig::db_dir_path_default(), - #[cfg(feature = "db-redis")] - redis_url: DbConfig::redis_url_default(), - #[cfg(feature = "db-mongo")] - mongodb_url: DbConfig::mongodb_url_default(), - } - } -} - -impl DbConfig { - fn login_prefix_default() -> String { - "ktra-secure-auth:".to_owned() - } - - #[cfg(feature = "db-sled")] - fn db_dir_path_default() -> PathBuf { - PathBuf::from("db") - } - - #[cfg(feature = "db-redis")] - fn redis_url_default() -> String { - "redis://localhost".to_owned() - } - - #[cfg(feature = "db-mongo")] - fn mongodb_url_default() -> String { - "mongodb://localhost:27017".to_owned() - } -} - #[derive(Debug, Clone, Deserialize)] pub struct ServerConfig { #[serde(default = "ServerConfig::address_default")] @@ -161,61 +123,12 @@ impl ServerConfig { #[allow(dead_code)] #[derive(Debug, Clone, Deserialize, Default)] pub struct OpenIdConfig { - pub(crate) issuer_url: String, - pub(crate) redirect_url: String, - pub(crate) client_id: String, - pub(crate) client_secret: String, - #[serde(default)] - pub(crate) additional_scopes: Vec, - pub(crate) gitlab_authorized_groups: Option>, - pub(crate) gitlab_authorized_users: Option>, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct Config { - #[serde(default)] - pub crate_files_config: CrateFilesConfig, - #[serde(default)] - pub db_config: DbConfig, - #[serde(default = "Config::index_config_default")] - pub index_config: IndexConfig, - #[serde(default)] - pub server_config: ServerConfig, + pub issuer_url: String, + pub redirect_url: String, + pub client_id: String, + pub client_secret: String, #[serde(default)] - pub openid_config: OpenIdConfig, -} - -impl Default for Config { - fn default() -> Config { - Config { - crate_files_config: Default::default(), - db_config: Default::default(), - index_config: Config::index_config_default(), - server_config: Default::default(), - openid_config: Default::default(), - } - } -} - -impl Config { - pub async fn open(path: impl AsRef) -> anyhow::Result { - let mut file = OpenOptions::new() - .read(true) - .open(path) - .map_ok(BufReader::new) - .await?; - let mut buf = String::new(); - file.read_to_string(&mut buf).await?; - - toml::from_str(&buf).map_err(Into::into) - } - - fn index_config_default() -> IndexConfig { - IndexConfig { - local_path: IndexConfig::local_path_default(), - branch: IndexConfig::branch_default(), - name: IndexConfig::name_default(), - ..Default::default() - } - } + pub additional_scopes: Vec, + pub gitlab_authorized_groups: Option>, + pub gitlab_authorized_users: Option>, } diff --git a/src/db_manager.rs b/src/db_manager/mod.rs similarity index 100% rename from src/db_manager.rs rename to src/db_manager/mod.rs diff --git a/src/db_manager/mongo_db_manager.rs b/src/db_manager/mongo_db_manager.rs index 13b6542..b16136f 100644 --- a/src/db_manager/mongo_db_manager.rs +++ b/src/db_manager/mongo_db_manager.rs @@ -1,6 +1,5 @@ #![cfg(feature = "db-mongo")] -use crate::config::DbConfig; use crate::error::Error; use crate::models::{Entry, Metadata, Query, Search, User}; use argon2::{self, hash_encoded, verify_encoded}; @@ -55,40 +54,6 @@ pub struct MongoDbManager { #[async_trait] impl DbManager for MongoDbManager { - #[tracing::instrument(skip(config))] - async fn new(config: &DbConfig) -> Result { - tracing::info!("connect to MongoDB server: {}", config.mongodb_url); - - let url = Url::parse(&config.mongodb_url).map_err(Error::UrlParsing)?; - let database_name = url - .path_segments() - .and_then(|s| s.last()) - .map(ToOwned::to_owned) - .unwrap_or_else(|| "ktra".to_owned()); - - let initialization = async { - let options = ClientOptions::parse(url.as_str()).await?; - let client = Client::with_options(options)?; - let db = client.database(&database_name); - let collection = db.collection(SCHEMA_VERSION_KEY); - - if collection.estimated_document_count(None).await? == 0 { - collection - .insert_one(doc! { "version": SCHEMA_VERSION }, None) - .await?; - } - - let db_manager = MongoDbManager { - client, - database_name, - login_prefix: config.login_prefix.clone(), - }; - Ok(db_manager) - }; - - initialization.map_err(Error::Db).await - } - async fn get_login_prefix(&self) -> Result<&str, Error> { Ok(&self.login_prefix) } @@ -546,10 +511,6 @@ impl DbManager for MongoDbManager { state: openidconnect::CsrfToken, nonce: openidconnect::Nonce, ) -> Result<(), Error> { - let collection = self - .client - .database(&self.database_name) - .collection(OAUTH_NONCES_KEY); let nonces_query_document = doc! {"state": state.secret().to_string() }; self.update_or_insert_one(OAUTH_NONCES_KEY, nonces_query_document, nonce) @@ -578,6 +539,40 @@ impl DbManager for MongoDbManager { } impl MongoDbManager { + #[tracing::instrument(skip(mongodb_url, login_prefix))] + pub async fn new(mongodb_url: String, login_prefix: String) -> Result { + tracing::info!("connect to MongoDB server: {}", mongodb_url); + + let url = Url::parse(&mongodb_url).map_err(Error::UrlParsing)?; + let database_name = url + .path_segments() + .and_then(|s| s.last()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| "ktra".to_owned()); + + let initialization = async { + let options = ClientOptions::parse(url.as_str()).await?; + let client = Client::with_options(options)?; + let db = client.database(&database_name); + let collection = db.collection(SCHEMA_VERSION_KEY); + + if collection.estimated_document_count(None).await? == 0 { + collection + .insert_one(doc! { "version": SCHEMA_VERSION }, None) + .await?; + } + + let db_manager = MongoDbManager { + client, + database_name, + login_prefix, + }; + Ok(db_manager) + }; + + initialization.map_err(Error::Db).await + } + #[tracing::instrument(skip(self, name, logins, editor))] async fn edit_owners(&self, name: N, logins: L, editor: E) -> Result<(), Error> where diff --git a/src/db_manager/redis_db_manager.rs b/src/db_manager/redis_db_manager.rs index b589a2f..42f90d2 100644 --- a/src/db_manager/redis_db_manager.rs +++ b/src/db_manager/redis_db_manager.rs @@ -1,6 +1,5 @@ #![cfg(feature = "db-redis")] -use crate::config::DbConfig; use crate::error::Error; use crate::models::{Entry, Metadata, Query, Search, User}; use argon2::{self, hash_encoded, verify_encoded}; @@ -32,28 +31,6 @@ pub struct RedisDbManager { #[async_trait] impl DbManager for RedisDbManager { - #[tracing::instrument(skip(config))] - async fn new(config: &DbConfig) -> Result { - tracing::info!("connect to redis server: {}", config.redis_url); - - let initialization = async { - let client = Client::open(&*config.redis_url)?; - let mut connection = client.get_async_connection().await?; - - if !connection.exists(SCHEMA_VERSION_KEY).await? { - connection.set(SCHEMA_VERSION_KEY, &SCHEMA_VERSION).await?; - } - - let db_manager = RedisDbManager { - client, - login_prefix: config.login_prefix.clone(), - }; - Ok(db_manager) - }; - - initialization.map_err(Error::Db).await - } - async fn get_login_prefix(&self) -> Result<&str, Error> { Ok(&self.login_prefix) } @@ -433,6 +410,28 @@ impl DbManager for RedisDbManager { } impl RedisDbManager { + #[tracing::instrument(skip(redis_url, login_prefix))] + pub async fn new(redis_url: String, login_prefix: String) -> Result { + tracing::info!("connect to redis server: {}", redis_url); + + let initialization = async { + let client = Client::open(&*redis_url)?; + let mut connection = client.get_async_connection().await?; + + if !connection.exists(SCHEMA_VERSION_KEY).await? { + connection.set(SCHEMA_VERSION_KEY, &SCHEMA_VERSION).await?; + } + + let db_manager = RedisDbManager { + client, + login_prefix, + }; + Ok(db_manager) + }; + + initialization.map_err(Error::Db).await + } + #[tracing::instrument(skip(self, name, logins, editor))] async fn edit_owners(&self, name: N, logins: L, editor: E) -> Result<(), Error> where diff --git a/src/db_manager/sled_db_manager.rs b/src/db_manager/sled_db_manager.rs index 67d7f13..5d4c2e5 100644 --- a/src/db_manager/sled_db_manager.rs +++ b/src/db_manager/sled_db_manager.rs @@ -1,6 +1,5 @@ #![cfg(feature = "db-sled")] -use crate::config::DbConfig; use crate::error::Error; use crate::models::{Entry, Metadata, Query, Search, User}; use argon2::{self, hash_encoded, verify_encoded}; @@ -11,6 +10,7 @@ use serde::de::DeserializeOwned; use serde::ser::Serialize; use sled::{self, Db}; use std::collections::HashMap; +use std::path::PathBuf; use crate::db_manager::utils::{argon2_config_and_salt, check_crate_name, normalized_crate_name}; use crate::db_manager::DbManager; @@ -22,6 +22,7 @@ const SCHEMA_VERSION: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 3]; const USERS_KEY: &str = "__USERS__"; const PASSWORDS_KEY: &str = "__PASSWORDS__"; const TOKENS_KEY: &str = "__TOKENS__"; +#[cfg(feature = "openid")] const OAUTH_NONCES_KEY: &str = "__OAUTH_NONCES__"; const OLD_TOKENS_KEY: &str = "tokens"; @@ -33,31 +34,6 @@ pub struct SledDbManager { #[async_trait] impl DbManager for SledDbManager { - #[tracing::instrument(skip(config))] - async fn new(config: &DbConfig) -> Result { - let path = config.db_dir_path.clone(); - tracing::info!("create and/or open database: {:?}", config.db_dir_path); - - let tree = tokio::task::spawn_blocking(|| sled::open(path).map_err(Error::Db)) - .map_err(Error::Join) - .await??; - Self::migrate_tokens(&tree).await?; - - if !tree.contains_key(SCHEMA_VERSION_KEY).map_err(Error::Db)? { - tree.insert(SCHEMA_VERSION_KEY, &SCHEMA_VERSION) - .map(drop) - .map_err(Error::Db)?; - tree.flush_async().map_err(Error::Db).await?; - } - - let db_manager = SledDbManager { - tree, - login_prefix: config.login_prefix.clone(), - }; - - Ok(db_manager) - } - async fn get_login_prefix(&self) -> Result<&str, Error> { Ok(&self.login_prefix) } @@ -434,6 +410,28 @@ impl DbManager for SledDbManager { } impl SledDbManager { + #[tracing::instrument(skip(db_dir_path, login_prefix))] + pub async fn new(db_dir_path: PathBuf, login_prefix: String) -> Result { + let path = db_dir_path; + tracing::info!("create and/or open database: {:?}", path.to_string_lossy()); + + let tree = tokio::task::spawn_blocking(move || sled::open(path).map_err(Error::Db)) + .map_err(Error::Join) + .await??; + Self::migrate_tokens(&tree).await?; + + if !tree.contains_key(SCHEMA_VERSION_KEY).map_err(Error::Db)? { + tree.insert(SCHEMA_VERSION_KEY, &SCHEMA_VERSION) + .map(drop) + .map_err(Error::Db)?; + tree.flush_async().map_err(Error::Db).await?; + } + + let db_manager = SledDbManager { tree, login_prefix }; + + Ok(db_manager) + } + #[tracing::instrument(skip(self, name, logins, editor))] async fn edit_owners(&self, name: N, logins: L, editor: E) -> Result<(), Error> where diff --git a/src/db_manager/traits.rs b/src/db_manager/traits.rs index 2a8c7ac..1531e72 100644 --- a/src/db_manager/traits.rs +++ b/src/db_manager/traits.rs @@ -1,4 +1,3 @@ -use crate::config::DbConfig; use crate::error::Error; use crate::models::{Metadata, Query, Search, User}; use async_trait::async_trait; @@ -6,7 +5,6 @@ use semver::Version; #[async_trait] pub trait DbManager: Send + Sync + Sized { - async fn new(confg: &DbConfig) -> Result; async fn get_login_prefix(&self) -> Result<&str, Error>; async fn can_edit_owners(&self, user_id: u32, name: &str) -> Result; diff --git a/src/error.rs b/src/error.rs index e1400fa..ed5695d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,7 @@ +use std::io; + use semver::Version; use serde::Serialize; -use std::io; use thiserror::Error; #[derive(Debug, Clone, Serialize)] diff --git a/src/get.rs b/src/get.rs deleted file mode 100644 index 76638f2..0000000 --- a/src/get.rs +++ /dev/null @@ -1,239 +0,0 @@ -use crate::db_manager::DbManager; -#[cfg(feature = "crates-io-mirroring")] -use crate::error::Error; -use crate::models::{Query, User}; -use crate::utils::*; -use futures::TryFutureExt; -#[cfg(feature = "crates-io-mirroring")] -use reqwest::Client; -#[cfg(feature = "crates-io-mirroring")] -use semver::Version; -use std::path::PathBuf; -use std::sync::Arc; -#[cfg(feature = "crates-io-mirroring")] -use tokio::fs::OpenOptions; -#[cfg(feature = "crates-io-mirroring")] -use tokio::io::{AsyncWriteExt, BufWriter}; -use tokio::{io::AsyncReadExt, sync::RwLock}; -#[cfg(feature = "crates-io-mirroring")] -use url::Url; -#[cfg(feature = "crates-io-mirroring")] -use warp::http::Response; -#[cfg(feature = "crates-io-mirroring")] -use warp::hyper::body::Bytes; -use warp::{filters::BoxedFilter, Filter, Rejection, Reply}; - -#[cfg(not(feature = "crates-io-mirroring"))] -#[tracing::instrument(skip(db_manager, dl_dir_path, path))] -pub fn apis( - db_manager: Arc>, - dl_dir_path: Arc, - path: Vec, -) -> impl Filter + Clone { - let routes = download(dl_dir_path, path) - .or(owners(db_manager.clone())) - .or(search(db_manager)); - - // With openid enabled, the `/me` route is handled in src/openid.rs - #[cfg(not(feature = "openid"))] - let routes = routes.or(me()); - - routes -} - -#[cfg(feature = "crates-io-mirroring")] -#[tracing::instrument(skip(db_manager, dl_dir_path, http_client, cache_dir_path, path))] -pub fn apis( - db_manager: Arc>, - dl_dir_path: Arc, - http_client: Client, - cache_dir_path: Arc, - path: Vec, -) -> impl Filter + Clone { - let routes = download(dl_dir_path, path) - .or(download_crates_io(http_client, cache_dir_path)) - .or(owners(db_manager.clone())) - .or(search(db_manager)); - // With openid enabled, the `/me` route is handled in src/openid.rs - #[cfg(not(feature = "openid"))] - let routes = routes.or(me()); - routes -} - -#[tracing::instrument(skip(path))] -pub(crate) fn into_boxed_filters(path: Vec) -> BoxedFilter<()> { - let (h, t) = path.split_at(1); - t.iter().fold(warp::path(h[0].clone()).boxed(), |accm, s| { - accm.and(warp::path(s.clone())).boxed() - }) -} - -#[tracing::instrument(skip(path, dl_dir_path))] -fn download( - dl_dir_path: Arc, - path: Vec, -) -> impl Filter + Clone { - into_boxed_filters(path).and(warp::fs::dir(dl_dir_path.to_path_buf())) -} - -#[cfg(feature = "crates-io-mirroring")] -#[tracing::instrument(skip(http_client, cache_dir_path, crate_name, version))] -async fn cache_crate_file( - http_client: Client, - cache_dir_path: Arc, - crate_name: impl AsRef, - version: Version, -) -> Result { - let computation = async move { - let mut cache_dir_path = cache_dir_path.as_ref().to_path_buf(); - let crate_components = format!("{}/{}/download", crate_name.as_ref(), version); - cache_dir_path.push(&crate_components); - let cache_file_path = cache_dir_path; - - if file_exists_and_not_empty(&cache_file_path).await { - OpenOptions::new() - .write(false) - .create(false) - .read(true) - .open(cache_file_path) - .and_then(|mut file| async move { - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer).await?; - Ok(Bytes::from(buffer)) - }) - .map_err(Error::Io) - .await - } else { - let mut crate_dir_path = cache_file_path.clone(); - crate_dir_path.pop(); - let crate_dir_path = crate_dir_path; - - tokio::fs::create_dir_all(crate_dir_path) - .map_err(Error::Io) - .await?; - - let file = OpenOptions::new() - .write(true) - .create(true) - .read(true) - .open(&cache_file_path) - .map_err(Error::Io) - .await?; - let mut file = BufWriter::with_capacity(128 * 1024, file); - - let crates_io_base_url = - Url::parse("https://crates.io/api/v1/crates/").map_err(Error::UrlParsing)?; - let crate_file_url = crates_io_base_url - .join(&crate_components) - .map_err(Error::UrlParsing)?; - let body = http_client - .get(crate_file_url) - .send() - .and_then(|res| async move { res.error_for_status() }) - .and_then(|res| res.bytes()) - .map_err(Error::HttpRequest) - .await?; - - if body.is_empty() { - return Err(Error::InvalidHttpResponseLength); - } - - file.write_all(&body).map_err(Error::Io).await?; - file.flush().map_err(Error::Io).await?; - - Ok(body) - } - }; - - computation.map_err(warp::reject::custom).await -} - -#[cfg(feature = "crates-io-mirroring")] -#[tracing::instrument(skip(cache_dir_path))] -fn download_crates_io( - http_client: Client, - cache_dir_path: Arc, -) -> impl Filter + Clone { - warp::get() - .and(with_http_client(http_client)) - .and(with_cache_dir_path(cache_dir_path)) - .and(warp::path!( - "ktra" / "api" / "v1" / "mirror" / String / Version / "download" - )) - .and_then(cache_crate_file) - .and_then(handle_download_crates_io) -} - -#[cfg(feature = "crates-io-mirroring")] -#[tracing::instrument(skip(crate_file_data))] -async fn handle_download_crates_io(crate_file_data: Bytes) -> Result { - let response = Response::builder() - .header("Content-Type", "application/x-tar") - .body(crate_file_data) - .map_err(Error::HttpResponseBuilding)?; - - Ok(response) -} - -#[tracing::instrument(skip(db_manager))] -fn owners( - db_manager: Arc>, -) -> impl Filter + Clone { - warp::get() - .and(with_db_manager(db_manager)) - .and(authorization_header()) - .and(warp::path!("api" / "v1" / "crates" / String / "owners")) - .and_then(handle_owners) -} - -#[tracing::instrument(skip(db_manager, _token, name))] -async fn handle_owners( - db_manager: Arc>, - // `token` is not a used argument. - // the specification demands that the authorization is required but listing owners api does not update the database. - _token: String, - name: String, -) -> Result { - let db_manager = db_manager.read().await; - let owners = db_manager - .owners(&name) - .map_err(warp::reject::custom) - .await?; - Ok(owners_json(owners)) -} - -#[tracing::instrument(skip(db_manager))] -fn search( - db_manager: Arc>, -) -> impl Filter + Clone { - warp::get() - .and(with_db_manager(db_manager)) - .and(warp::path!("api" / "v1" / "crates")) - .and(warp::query::()) - .and_then(handle_search) -} - -#[tracing::instrument(skip(db_manager, query))] -async fn handle_search( - db_manager: Arc>, - query: Query, -) -> Result { - let db_manager = db_manager.read().await; - db_manager - .search(&query) - .map_ok(|s| warp::reply::json(&s)) - .map_err(warp::reject::custom) - .await -} - -#[tracing::instrument] -fn me() -> impl Filter + Clone { - warp::get() - .and(warp::path!("me")) - .map(|| "$ curl -X POST -H 'Content-Type: application/json' -d '{\"password\":\"YOUR PASSWORD\"}' https:///ktra/api/v1/login/") -} - -#[tracing::instrument(skip(owners))] -fn owners_json(owners: Vec) -> impl Reply { - warp::reply::json(&serde_json::json!({ "users": owners })) -} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..1a6880e --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,9 @@ +pub mod apis; +pub mod config; +pub mod db_manager; +pub mod error; +mod index_manager; +pub mod models; +pub mod utils; + +pub use index_manager::IndexManager; diff --git a/src/models.rs b/src/models.rs index f58ad23..e9dc097 100644 --- a/src/models.rs +++ b/src/models.rs @@ -302,23 +302,3 @@ pub struct ChangePassword { pub old_password: String, pub new_password: String, } - -#[cfg(feature = "openid")] -#[derive(Debug, Clone, Deserialize)] -pub struct CodeQuery { - pub code: String, - pub state: Option, -} - -/// The additional claims OpenId providers may send -/// -/// All fields here are options so that the extra claims are caught when presents -#[cfg(feature = "openid")] -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct Claims { - pub(crate) sub: Option, - pub(crate) sub_legacy: Option, - // Gitlab claims return the groups a user is in. - // This property is used when gitlab_authorized_groups is set in the configuration - pub(crate) groups: Option>, -} diff --git a/src/utils.rs b/src/utils.rs index a466cf8..0c67ef3 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,3 +1,4 @@ +//! Utility functions use crate::config::OpenIdConfig; use crate::db_manager::DbManager; use crate::error::Error; @@ -5,14 +6,13 @@ use crate::index_manager::IndexManager; use futures::TryFutureExt; use rand::distributions::Alphanumeric; use rand::prelude::*; -#[cfg(feature = "crates-io-mirroring")] -use reqwest::Client; use std::convert::Infallible; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::sync::RwLock; use warp::{Filter, Rejection, Reply}; +#[allow(dead_code)] #[inline] pub fn always_true(_: T) -> bool { true @@ -70,6 +70,23 @@ pub fn ok_with_msg_json_message(msg: impl Into) -> impl Reply { })) } +#[tracing::instrument(skip(rejection))] +pub async fn handle_rejection(rejection: Rejection) -> Result { + if let Some(application_error) = rejection.find::() { + let (json, status_code) = application_error.to_reply(); + Ok(warp::reply::with_status(json, status_code)) + } else { + Ok(warp::reply::with_status( + warp::reply::json(&serde_json::json!({ + "errors": [ + { "detail": "resource or api is not defined" } + ] + })), + warp::http::StatusCode::NOT_FOUND, + )) + } +} + #[tracing::instrument(skip(dl_dir_path))] pub fn with_dl_dir_path( dl_dir_path: Arc, @@ -88,8 +105,8 @@ pub fn with_cache_dir_path( #[cfg(feature = "crates-io-mirroring")] #[tracing::instrument(skip(client))] pub fn with_http_client( - client: Client, -) -> impl Filter + Clone { + client: reqwest::Client, +) -> impl Filter + Clone { warp::any().map(move || client.clone()) } @@ -124,7 +141,7 @@ mod tests { use super::package_dir_path; #[test] - fn test_package_dir_path_a() -> anyhow::Result<()> { + fn test_package_dir_path_a() -> Result<(), crate::error::Error> { let dir = package_dir_path("a")?; assert_eq!(dir.as_ref().to_str().unwrap(), "1"); @@ -132,7 +149,7 @@ mod tests { } #[test] - fn test_package_dir_path_ab() -> anyhow::Result<()> { + fn test_package_dir_path_ab() -> Result<(), crate::error::Error> { let dir = package_dir_path("ab")?; assert_eq!(dir.as_ref().to_str().unwrap(), "2"); @@ -140,7 +157,7 @@ mod tests { } #[test] - fn test_package_dir_path_abc() -> anyhow::Result<()> { + fn test_package_dir_path_abc() -> Result<(), crate::error::Error> { let dir = package_dir_path("abc")?; assert_eq!(dir.as_ref().to_str().unwrap(), "3/a"); @@ -148,7 +165,7 @@ mod tests { } #[test] - fn test_package_dir_path_abcd() -> anyhow::Result<()> { + fn test_package_dir_path_abcd() -> Result<(), crate::error::Error> { let dir = package_dir_path("abcd")?; assert_eq!(dir.as_ref().to_str().unwrap(), "ab/cd"); diff --git a/test-awrp/Cargo.toml b/test-awrp/Cargo.toml new file mode 100644 index 0000000..7079b6a --- /dev/null +++ b/test-awrp/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "test-awrp" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { version = "1.1", features = ["macros", "rt-multi-thread", "fs", "io-util"] } +warp = "0.3" +serde_json = "1" diff --git a/test-awrp/src/main.rs b/test-awrp/src/main.rs new file mode 100644 index 0000000..dea4456 --- /dev/null +++ b/test-awrp/src/main.rs @@ -0,0 +1,23 @@ +use warp::Filter; + +#[tokio::main] +async fn main() { + warp::serve( + warp::path("ttt").and( +// warp::path(".git").and(warp::path::tail()).map(|tail|{ +// let d = format!("resource or api {:?} is not defined", tail); +// warp::reply::json(&serde_json::json!({ +// "errors": [ +// { "detail": d } +// ] +// }))}) + warp::path!(".git/aaa").and(warp::path::tail()).map(|tail| format!("{:?}", tail)) + .or( + warp::fs::dir("../index") + ) + ) + ) + .run(([127, 0, 0, 1], 3030)) + .await; +} +