From afb9f52cfe1c4854391eccb263781e6a1ed4569a Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:50:00 +0100 Subject: [PATCH 01/26] Add consume batching delay --- apps/labrinth/.env.docker-compose | 1 + apps/labrinth/.env.local | 1 + apps/labrinth/src/env.rs | 1 + apps/labrinth/src/search/incremental.rs | 41 +++---- .../src/search/incremental/consume.rs | 101 +++++++++++++----- 5 files changed, 91 insertions(+), 54 deletions(-) diff --git a/apps/labrinth/.env.docker-compose b/apps/labrinth/.env.docker-compose index 097a47c640..288e1aa0fc 100644 --- a/apps/labrinth/.env.docker-compose +++ b/apps/labrinth/.env.docker-compose @@ -25,6 +25,7 @@ ELASTICSEARCH_INDEX_PREFIX=labrinth ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=elastic SEARCH_INDEX_CHUNK_SIZE=5000 +SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5 TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth diff --git a/apps/labrinth/.env.local b/apps/labrinth/.env.local index 7925e52a1e..7843c066f8 100644 --- a/apps/labrinth/.env.local +++ b/apps/labrinth/.env.local @@ -43,6 +43,7 @@ ELASTICSEARCH_USERNAME= ELASTICSEARCH_PASSWORD= SEARCH_INDEX_CHUNK_SIZE=5000 +SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5 TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index bf99822156..3001a40d0c 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -160,6 +160,7 @@ vars! { // search SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense; SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64; + SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS: u64 = 5u64; TYPESENSE_URL: String = "http://localhost:8108"; TYPESENSE_API_KEY: String = "modrinth"; TYPESENSE_INDEX_PREFIX: String = "labrinth"; diff --git a/apps/labrinth/src/search/incremental.rs b/apps/labrinth/src/search/incremental.rs index 07c5a05b55..88a9f1d6a2 100644 --- a/apps/labrinth/src/search/incremental.rs +++ b/apps/labrinth/src/search/incremental.rs @@ -1,6 +1,6 @@ pub mod consume; -use std::{mem, sync::Arc}; +use std::{collections::HashSet, mem, sync::Arc, time::Duration}; use rdkafka::{producer::FutureRecord, util::Timeout}; use serde::Serialize; @@ -13,31 +13,29 @@ use crate::{ pub const SEARCH_PROJECT_INDEX_QUEUE_TOPIC: &str = "public.labrinth.search-project-index-queue.v1"; +const QUEUE_FLUSH_INTERVAL: Duration = Duration::from_secs(10); #[derive(Clone)] pub struct IncrementalSearchQueue { - operations: Arc>>, + project_ids: Arc>>, kafka_client: actix_web::web::Data, } impl IncrementalSearchQueue { pub fn new(kafka_client: actix_web::web::Data) -> Self { Self { - operations: Arc::new(Mutex::new(Vec::new())), + project_ids: Arc::new(Mutex::new(HashSet::new())), kafka_client, } } pub async fn push(&self, project_id: ProjectId) { - self.operations - .lock() - .await - .push(SearchIndexOperation { project_id }); + self.project_ids.lock().await.insert(project_id); } pub async fn run(self) { loop { - tokio::time::sleep(KAFKA_OPERATION_INTERVAL).await; + tokio::time::sleep(QUEUE_FLUSH_INTERVAL).await; if let Err(err) = self.drain().await { tracing::error!( @@ -48,22 +46,20 @@ impl IncrementalSearchQueue { } pub async fn drain(&self) -> eyre::Result<()> { - let operations = { - let mut operations = self.operations.lock().await; - mem::take(&mut *operations) + let project_ids = { + let mut project_ids = self.project_ids.lock().await; + mem::take(&mut *project_ids) }; - if operations.is_empty() { + if project_ids.is_empty() { return Ok(()); } - let mut operations = operations.into_iter(); - while let Some(operation) = operations.next() { + let mut project_ids = project_ids.into_iter(); + while let Some(project_id) = project_ids.next() { let event = KafkaEvent::new( SEARCH_PROJECT_INDEX_QUEUE_TOPIC, - SearchProjectIndexQueueEventData { - project_id: operation.project_id, - }, + SearchProjectIndexQueueEventData { project_id }, ); let event_id = event.event_metadata.event_id; let key = event_id.to_string(); @@ -78,9 +74,9 @@ impl IncrementalSearchQueue { .send(record, Timeout::After(KAFKA_OPERATION_INTERVAL)) .await { - let mut queued_operations = self.operations.lock().await; - queued_operations.push(operation); - queued_operations.extend(operations); + let mut queued_project_ids = self.project_ids.lock().await; + queued_project_ids.insert(project_id); + queued_project_ids.extend(project_ids); return Err(err.into()); } @@ -90,11 +86,6 @@ impl IncrementalSearchQueue { } } -#[derive(Debug, Clone)] -pub struct SearchIndexOperation { - pub project_id: ProjectId, -} - #[derive(Debug, Serialize)] pub struct SearchProjectIndexQueueEventData { pub project_id: ProjectId, diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 69e3b30f54..040285a895 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -1,16 +1,21 @@ use actix_web::web; use eyre::WrapErr; -use futures::FutureExt; +use futures::never::Never; use rdkafka::{ Message, consumer::{CommitMode, Consumer, StreamConsumer}, message::BorrowedMessage, }; use serde::Deserialize; -use std::collections::HashSet; +use std::{ + collections::HashSet, + time::{Duration, Instant}, +}; +use tracing::info; use crate::{ database::{PgPool, redis::RedisPool}, + env::ENV, models::ids::ProjectId, search::{ SearchBackend, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC, @@ -22,8 +27,6 @@ use crate::{ }, }; -const BATCH_SIZE: usize = 100; - pub async fn run( ro_pool: PgPool, redis_pool: RedisPool, @@ -60,25 +63,57 @@ async fn consume( search_backend: &dyn SearchBackend, consumer: &StreamConsumer, ) -> eyre::Result<()> { + // keep buffer capacity (pre-)allocated + let mut messages = Vec::with_capacity(1024); loop { - let mut messages = Vec::with_capacity(BATCH_SIZE); - messages.push( - consumer - .recv() - .await - .wrap_err("failed to receive Kafka message")?, - ); + messages.clear(); - while messages.len() < BATCH_SIZE { - let Some(message) = consumer.recv().now_or_never() else { - break; - }; + // wait for a first message to come in... + let first_message = consumer + .recv() + .await + .wrap_err("failed to receive Kafka message")?; + messages.push(first_message); - messages.push(message.wrap_err("failed to receive Kafka message")?); + let delay = Duration::from_secs( + ENV.SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS, + ); + info!( + "Received initial Kafka message; waiting {delay:.2?} for more to batch", + ); + + // ..then wait a while for more messages to batch up + // so that we can process a big batch to reindex + // + // do a little trick with an `AsyncFnMut` closure + // so that we can explicitly specify the return type + let mut collect_more_messages = async || -> eyre::Result { + loop { + let message = consumer + .recv() + .await + .wrap_err("failed to receive Kafka message")?; + messages.push(message); + } + }; + match tokio::time::timeout(delay, collect_more_messages()).await { + Err(_elapsed) => {} + Ok(Err(err)) => { + return Err( + err.wrap_err("failed to receive more Kafka messages") + ); + } } - consume_batch(ro_pool, redis_pool, search_backend, consumer, messages) - .await?; + info!("Consuming batch of {} messages", messages.len()); + consume_batch( + ro_pool, + redis_pool, + search_backend, + consumer, + messages.drain(..), + ) + .await?; } } @@ -87,8 +122,10 @@ async fn consume_batch( redis_pool: &RedisPool, search_backend: &dyn SearchBackend, consumer: &StreamConsumer, - messages: Vec>, + messages: impl IntoIterator>, ) -> eyre::Result<()> { + let start = Instant::now(); + let mut project_ids = Vec::new(); let mut seen_project_ids = HashSet::new(); let mut messages_to_commit = Vec::new(); @@ -131,19 +168,19 @@ async fn consume_batch( messages_to_commit.push(message); } - if project_ids.is_empty() { - return Ok(()); - } - - tracing::info!( + info!( kafka.message_count = messages_to_commit.len(), - project_count = project_ids.len(), - "Consumed incremental search index event batch" + "Read all Kafka messages in {:.2?}, found {} projects to reindex", + start.elapsed(), + project_ids.len(), ); + let start = Instant::now(); - reindex_projects(ro_pool, redis_pool, search_backend, &project_ids) - .await - .wrap_err("failed to reindex project batch")?; + if !project_ids.is_empty() { + reindex_projects(ro_pool, redis_pool, search_backend, &project_ids) + .await + .wrap_err("failed to reindex project batch")?; + } for message in messages_to_commit { consumer @@ -151,6 +188,12 @@ async fn consume_batch( .wrap_err("failed to commit Kafka message")?; } + info!( + "Reindexed {} projects in {:.2?}", + project_ids.len(), + start.elapsed() + ); + Ok(()) } From 7821b6c1cd6421cddbf4f8337fa546b8378c6915 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:04:11 +0100 Subject: [PATCH 02/26] maybe fix --- apps/labrinth/src/background_task.rs | 3 +- .../src/search/incremental/consume.rs | 23 +++++++-------- apps/labrinth/src/search/indexing.rs | 29 ++++++++++--------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index 31c51e2367..0c3fb13ef0 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -19,7 +19,7 @@ use crate::util::anrok; use actix_web::web; use clap::ValueEnum; use eyre::WrapErr; -use tracing::info; +use tracing::{info, instrument}; #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)] #[clap(rename_all = "kebab_case")] @@ -50,6 +50,7 @@ pub enum BackgroundTask { impl BackgroundTask { #[allow(clippy::too_many_arguments)] + #[instrument(skip_all, fields(background_task = ?self))] pub async fn run( self, pool: PgPool, diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 040285a895..cb0d4e1a0e 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -11,7 +11,7 @@ use std::{ collections::HashSet, time::{Duration, Instant}, }; -use tracing::info; +use tracing::{Instrument, info, info_span}; use crate::{ database::{PgPool, redis::RedisPool}, @@ -214,18 +214,15 @@ pub async fn reindex_projects( ) -> eyre::Result<()> { search_backend.remove_project_documents(project_ids).await?; - let mut documents = Vec::new(); - for project_id in project_ids { - documents.extend( - index_project_documents(ro_pool, redis_pool, *project_id) - .await - .wrap_err_with(|| { - format!( - "failed to build project {project_id} search documents" - ) - })?, - ); - } + let documents = index_project_documents(ro_pool, redis_pool, project_ids) + .instrument(info_span!("index", batch_size = project_ids.len())) + .await + .wrap_err_with(|| { + format!( + "failed to build search documents for {} projects", + project_ids.len() + ) + })?; search_backend.index_documents(&documents).await?; diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 3c4c8a762f..75bfa3e926 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -7,7 +7,7 @@ use itertools::Itertools; use regex::Regex; use std::collections::HashMap; use std::sync::LazyLock; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use crate::database::PgPool; use crate::database::models::loader_fields::{ @@ -121,10 +121,13 @@ pub async fn index_local( pub async fn index_project_documents( pool: &PgPool, redis: &RedisPool, - project_id: ProjectId, + project_ids: &[ProjectId], ) -> eyre::Result> { let searchable_statuses = searchable_statuses(); - let project_ids = vec![DBProjectId::from(project_id).0]; + let project_ids = project_ids + .iter() + .map(|project_id| DBProjectId::from(*project_id).0) + .collect::>(); let db_projects = sqlx::query!( r#" @@ -177,7 +180,7 @@ async fn build_search_documents( .await .wrap_err("failed to fetch query context")?; - info!("Indexing local dependencies!"); + debug!("Indexing local dependencies!"); let dependencies: DashMap> = sqlx::query!( @@ -231,7 +234,7 @@ async fn build_search_documents( ordering: i64, } - info!("Indexing local gallery!"); + debug!("Indexing local gallery!"); let mods_gallery: DashMap> = sqlx::query!( " @@ -257,7 +260,7 @@ async fn build_search_documents( ) .await?; - info!("Indexing local categories!"); + debug!("Indexing local categories!"); let categories: DashMap> = sqlx::query!( " @@ -280,10 +283,10 @@ async fn build_search_documents( ) .await?; - info!("Indexing local versions!"); + debug!("Indexing local versions!"); let mut versions = index_versions(pool, project_ids.clone()).await?; - info!("Indexing local org owners!"); + debug!("Indexing local org owners!"); let mods_org_owners: DashMap = sqlx::query!( " @@ -308,7 +311,7 @@ async fn build_search_documents( }) .await?; - info!("Indexing local team owners!"); + debug!("Indexing local team owners!"); let mods_team_owners: DashMap = sqlx::query!( " @@ -332,7 +335,7 @@ async fn build_search_documents( }) .await?; - info!("Getting all loader fields!"); + debug!("Getting all loader fields!"); let loader_fields: Vec = sqlx::query!( " SELECT DISTINCT id, field, field_type, enum_type, min_val, max_val, optional @@ -353,7 +356,7 @@ async fn build_search_documents( .await?; let loader_fields: Vec<&QueryLoaderField> = loader_fields.iter().collect(); - info!("Getting all loader field enum values!"); + debug!("Getting all loader field enum values!"); let loader_field_enum_values: Vec = sqlx::query!( @@ -375,7 +378,7 @@ async fn build_search_documents( .try_collect() .await?; - info!("Indexing loaders, project types!"); + debug!("Indexing loaders, project types!"); let mut uploads = Vec::new(); let total_len = db_projects.len(); @@ -384,7 +387,7 @@ async fn build_search_documents( count += 1; if count % 1000 == 0 { - info!("projects index prog: {count}/{total_len}"); + debug!("projects index prog: {count}/{total_len}"); } let Some(( _, From 003ceb991822b3c31cd46ed469b92c334aa886db Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:24:10 +0100 Subject: [PATCH 03/26] max batch size --- apps/labrinth/.env.docker-compose | 1 + apps/labrinth/.env.local | 1 + apps/labrinth/src/env.rs | 3 ++- .../src/search/incremental/consume.rs | 13 +++++++---- apps/labrinth/src/search/indexing.rs | 22 ++++++++++--------- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/apps/labrinth/.env.docker-compose b/apps/labrinth/.env.docker-compose index 288e1aa0fc..b900b2ec10 100644 --- a/apps/labrinth/.env.docker-compose +++ b/apps/labrinth/.env.docker-compose @@ -26,6 +26,7 @@ ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=elastic SEARCH_INDEX_CHUNK_SIZE=5000 SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5 +SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000 TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth diff --git a/apps/labrinth/.env.local b/apps/labrinth/.env.local index 7843c066f8..df34775b5e 100644 --- a/apps/labrinth/.env.local +++ b/apps/labrinth/.env.local @@ -44,6 +44,7 @@ ELASTICSEARCH_PASSWORD= SEARCH_INDEX_CHUNK_SIZE=5000 SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS=5 +SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000 TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index 3001a40d0c..69c5e8e7e6 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -38,7 +38,7 @@ macro_rules! vars { )] let $field: Option<$ty> = { let mut default = None::<$ty>; - $( default = Some({ $default }.into()); )? + $( default = Some(<$ty>::from({ $default })); )? match parse_value::<$ty>(stringify!($field), default) { Ok(value) => Some(value), @@ -161,6 +161,7 @@ vars! { SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense; SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64; SEARCH_INCREMENTAL_INDEX_BATCH_DELAY_SECONDS: u64 = 5u64; + SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE: usize = 1000usize; TYPESENSE_URL: String = "http://localhost:8108"; TYPESENSE_API_KEY: String = "modrinth"; TYPESENSE_INDEX_PREFIX: String = "labrinth"; diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index cb0d4e1a0e..f74b0dd5c1 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -83,21 +83,24 @@ async fn consume( ); // ..then wait a while for more messages to batch up - // so that we can process a big batch to reindex + // so that we can process a big batch to reindex. + // we stop until either we've reached the max batch size, + // or we've waited enough time - whichever is first. // // do a little trick with an `AsyncFnMut` closure // so that we can explicitly specify the return type - let mut collect_more_messages = async || -> eyre::Result { - loop { + let mut collect_more_messages = async || -> eyre::Result<()> { + while messages.len() < ENV.SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE { let message = consumer .recv() .await .wrap_err("failed to receive Kafka message")?; messages.push(message); } + eyre::Ok(()) }; match tokio::time::timeout(delay, collect_more_messages()).await { - Err(_elapsed) => {} + Ok(Ok(())) | Err(_) => {} Ok(Err(err)) => { return Err( err.wrap_err("failed to receive more Kafka messages") @@ -224,6 +227,8 @@ pub async fn reindex_projects( ) })?; + info!("Fetched all project documents, indexing into backend"); + search_backend.index_documents(&documents).await?; Ok(()) diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 75bfa3e926..77e334cee0 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -161,6 +161,8 @@ pub async fn index_project_documents( .await .wrap_err("failed to fetch project")?; + info!("Fetched partial projects"); + build_search_documents(pool, redis, db_projects).await } @@ -180,7 +182,7 @@ async fn build_search_documents( .await .wrap_err("failed to fetch query context")?; - debug!("Indexing local dependencies!"); + info!("Indexing local dependencies!"); let dependencies: DashMap> = sqlx::query!( @@ -234,7 +236,7 @@ async fn build_search_documents( ordering: i64, } - debug!("Indexing local gallery!"); + info!("Indexing local gallery!"); let mods_gallery: DashMap> = sqlx::query!( " @@ -260,7 +262,7 @@ async fn build_search_documents( ) .await?; - debug!("Indexing local categories!"); + info!("Indexing local categories!"); let categories: DashMap> = sqlx::query!( " @@ -283,10 +285,10 @@ async fn build_search_documents( ) .await?; - debug!("Indexing local versions!"); + info!("Indexing local versions!"); let mut versions = index_versions(pool, project_ids.clone()).await?; - debug!("Indexing local org owners!"); + info!("Indexing local org owners!"); let mods_org_owners: DashMap = sqlx::query!( " @@ -311,7 +313,7 @@ async fn build_search_documents( }) .await?; - debug!("Indexing local team owners!"); + info!("Indexing local team owners!"); let mods_team_owners: DashMap = sqlx::query!( " @@ -335,7 +337,7 @@ async fn build_search_documents( }) .await?; - debug!("Getting all loader fields!"); + info!("Getting all loader fields!"); let loader_fields: Vec = sqlx::query!( " SELECT DISTINCT id, field, field_type, enum_type, min_val, max_val, optional @@ -356,7 +358,7 @@ async fn build_search_documents( .await?; let loader_fields: Vec<&QueryLoaderField> = loader_fields.iter().collect(); - debug!("Getting all loader field enum values!"); + info!("Getting all loader field enum values!"); let loader_field_enum_values: Vec = sqlx::query!( @@ -378,7 +380,7 @@ async fn build_search_documents( .try_collect() .await?; - debug!("Indexing loaders, project types!"); + info!("Indexing loaders, project types!"); let mut uploads = Vec::new(); let total_len = db_projects.len(); @@ -387,7 +389,7 @@ async fn build_search_documents( count += 1; if count % 1000 == 0 { - debug!("projects index prog: {count}/{total_len}"); + info!("projects index prog: {count}/{total_len}"); } let Some(( _, From d4a7b15b0d8b217012aae16f106da06d71f928c8 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:39:13 +0100 Subject: [PATCH 04/26] more logging --- .../src/search/backend/typesense/mod.rs | 16 ++- .../src/search/incremental/consume.rs | 3 +- .../create-dummy-projects.cpython-314.pyc | Bin 0 -> 5414 bytes scripts/create-dummy-projects.py | 123 ++++++++++++++++++ 4 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 scripts/__pycache__/create-dummy-projects.cpython-314.pyc create mode 100755 scripts/create-dummy-projects.py diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index 6bbb141bf7..42beb818a5 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -7,7 +7,7 @@ use regex::Regex; use reqwest::Method; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use crate::database::PgPool; use crate::database::redis::RedisPool; @@ -1038,21 +1038,35 @@ impl SearchBackend for Typesense { self.config.get_alias_name("projects"), self.config.get_alias_name("projects_filtered"), ] { + debug!("Performing removal on alias {alias:?}"); + let live = self.client.get_alias(&alias).await?; + debug!("Got live alias {live:?}"); + let shadow_alt = self.config.get_next_collection_name(&alias, true); + debug!("Got shadow alt {shadow_alt:?}"); + let shadow_current = self.config.get_next_collection_name(&alias, false); + debug!("Got shadow current {shadow_current:?}"); for collection in live.into_iter().chain([shadow_alt, shadow_current]) { + debug!("Working on collection {collection:?}"); if self.client.collection_exists(&collection).await? { + debug!( + filter_len = filter.len(), + "Collection exists, deleting by filter" + ); self.client .delete_documents_by_filter(&collection, &filter) .await?; } } } + + debug!("Done"); Ok(()) } diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index f74b0dd5c1..58e8a6456a 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -1,6 +1,5 @@ use actix_web::web; use eyre::WrapErr; -use futures::never::Never; use rdkafka::{ Message, consumer::{CommitMode, Consumer, StreamConsumer}, @@ -215,8 +214,10 @@ pub async fn reindex_projects( search_backend: &dyn SearchBackend, project_ids: &[ProjectId], ) -> eyre::Result<()> { + info!("Removing documents for batch"); search_backend.remove_project_documents(project_ids).await?; + info!("Creating project documents"); let documents = index_project_documents(ro_pool, redis_pool, project_ids) .instrument(info_span!("index", batch_size = project_ids.len())) .await diff --git a/scripts/__pycache__/create-dummy-projects.cpython-314.pyc b/scripts/__pycache__/create-dummy-projects.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5100c7d3103b7fd09b5ee18fc966255066873c37 GIT binary patch literal 5414 zcma)AYit|G5#De7}6>&p%KQI z)JUVHG}>qxqb0}Klbpt#NkJ1lNTlnLW)&HzZ6XV`{TYYI9r!LnaX3da zddA*{`m5UlH_>kV_wk=%+F2i|yMj}S6paTmndGsnQY@=w6x|k8Be6_MPD+{-_Ypdi z%_idgF{&yDH?{B|k01RUsyq@wY={&|JK9T%REQ91kr7#u6Zu{wT0}v#LTeN4eY{MC zVMV0`Oof;PXxmVT66-|AV0BKW#hMVZA+a_@L+v#BuDx7YNJns4|_~HR_EUEV7chB*irEL`Lx@GqGqAYFMO6 zs+LGk_UhKSq{ftltR*t(+`-^fLiL(Uu0&Puq?DGFDCn!QI5}+yx){}BQ{F^6E?os7 zhA_2PuT7*AS|XZ^JS!OwLRlhUdbNyK&P%vcb< zg`i4u6gJ8zYL4{#bKkKGM>1(mN^Ab3302OhAUTsB!n>#Z@u(IZ_F_dG+K&6%*O-IT zvNYt4$}(&b#Y_4wshPB0unRfbE`VU#@3#x^K!_fl81hj%jpua+lqIV=E2U$Zxa4DW z{$eJZj)Nu@3w#BgG486QsiPLKCLvvGO!_}=Ap`?`y@YmYL1zU6|NP9_7aSClt^pQK2RcSiq3N-(p|V% zB5kwr66q+AxMIpqi-pR&SGFQs?Tw+I4He?^xx3cA`TF}bo4=aBR7Rw;yUZfio*yeC z!oiien*5~#Tei?t!(#!_!py)=4wUUIX?p}`Krt5A)xtw-*uaxe=i%ML`$QpHB)|$N znGF*rke$f~1} zts!f`zF`pzP5T5#B%Hq-aD0_RwCJSaPEY&6O-UMno$B)v>=c1cjm2Oss$y0Yu#P32is5rt_=hjvM7E5%ad(E+c(`Qu;r_3 zMZ4$_YfL@Kc95`j6Rz3=uCRfmAYV#YU^QQC!s%SKfP1GoXRwQKJdXho04E79P2nOk&-f5 zaKz|rj>&2h{sTIvNOCe7lVF;a6a`#wVg*d{xXu_O-I`5^DXqZcLOLre z(nR8_f?o+foQC5~#f8&`@7-=pL^VxH$(l~V@$rr-Yyx+NZKbQ4iVYaty5ZTqMp5v4 z(j67&9;xbFISNGl5L6YkmPaM4bGEb4Rya1-UKlR&p1k`W?<{oA?_AA4fdI&kl`Mfcu?0}EY6*P${)*{FQ|ih!J33zPH2&CBoCZqJ{;$2)E~UT`e)t&4nX ziFM?^H+!u}HI;Z<$?7TzwPg#kcpf3fLg(Ad4uR_a(t-x1j}?WcC1D`nUt%rSr)H)Kb z7qD@SUz91}`Fw>(6nDMr$6YTyRd5zg+}t)NF4?`yWSjalDF4qbz9TO39oBPrgnMVF zfcqoXBQ@4L0uB8;H59I0o?{1?J9|diV>I*bAOVB-XdAAbM&IQ=wvTzQ!|3m0fF1+E ztEC2dIEFo%h3Jw;--h$dXdt5y+6>KMOevD4_w;-n7X?xD+qszVYNW)tbg&w0zg8;0 z#y?K}AN-be{LBXaP2%zD-3d{UNzobx?_VilBYnXAz-q5L1kO(L9m)>`y5mnC~4 znjb$1IED+c0qll1lN`e|2x7D_32(U>oeClp?lvVt zwq!A~C7d(4!hVw;VoYi9{aMJf+-vrADqGQwxCQ^MVXrl*;a*ctV7J*@Yuk{x6>_)w z&EJ670LZv;vy+IR3$bbao%i{D=jrnz=MlvF0JB+}Z{mlGYo2~Kj>N&nK zIU+NW%&Jpo6)-Zu64e+k#dWUAuCo3@)wt8g)LZOaiJ>CEYQq zOlDI+4d5aEfgm1_z^p;jDY&367@U}>Dgz)&!8xB!O-WZ3>}z%FIxQ((n8Fw;1>lpt zm`Z2{NKl6G%3)j`!Ymdk4O`%}S^?7n)B;C^Z=|!3w@T@_&d6A~I+X-igU_#EN93yk zbW=GeZ5#&!o{BEW%w^>q7Q{={-#V%QXd^GUy-gUzp*7<7Z>bBT263Ro11f4<GQ%en<#d_au=Ue-KJNWzIH-{IL-;ETzhKs_%{MP#{mv@zH zj(o!>Y{PtTdFTGco%@&Bfq$?AZ+AjE4G=8;%B7bt-K}kVw8d(tAGncodp=vPL$vMs z$jrz++J&*ynVB=Vd2;5-5^XEdLW#EE@#&eZF~O2%;Ed6+qs!@h2VVu zZTiN7-7-*NCdy0NW@p8h_I~ksO-fHFGSG1NS4W^K zQv^YLjA}kaR^wmYhiJ=3Xyl)0>qEAVus!T)CE6bzCiW9lStj5$%;xS=jUN!qPq#jx J5WUUV;C~)fnq&X~ literal 0 HcmV?d00001 diff --git a/scripts/create-dummy-projects.py b/scripts/create-dummy-projects.py new file mode 100755 index 0000000000..c55ea25d2e --- /dev/null +++ b/scripts/create-dummy-projects.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +import argparse +import json +import time +import urllib.error +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from uuid import uuid4 + + +def make_body(boundary, slug, index): + data = { + "name": f"Dummy Load {index:04d}", + "slug": slug, + "summary": "A dummy project for local load testing.", + "description": "This project was generated locally for batch indexing tests.", + "initial_versions": [], + "is_draft": True, + "categories": [], + "license_id": "MIT", + } + + payload = json.dumps(data, separators=(",", ":")) + return ( + f"--{boundary}\r\n" + 'Content-Disposition: form-data; name="data"\r\n' + "Content-Type: application/json\r\n\r\n" + f"{payload}\r\n" + f"--{boundary}--\r\n" + ).encode() + + +def create_project(base_url, token, boundary, prefix, index, retries): + slug = f"{prefix}-{index:04d}" + body = make_body(boundary, slug, index) + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": f"multipart/form-data; boundary={boundary}", + } + + for attempt in range(retries + 1): + req = urllib.request.Request( + f"{base_url}/v3/project", + data=body, + headers=headers, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=60) as resp: + resp.read() + return True, slug, resp.status, "" + except urllib.error.HTTPError as err: + text = err.read().decode("utf-8", errors="replace") + if err.code < 500 or attempt == retries: + return False, slug, err.code, text + except Exception as err: + if attempt == retries: + return False, slug, "error", repr(err) + + time.sleep(min(2**attempt, 10)) + + raise RuntimeError("unreachable") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8000") + parser.add_argument("--token", default="mra_admin") + parser.add_argument("--count", type=int, default=1000) + parser.add_argument("--concurrency", type=int, default=2) + parser.add_argument("--retries", type=int, default=5) + args = parser.parse_args() + + boundary = "----modrinth-dummy-project-boundary" + prefix = f"dummy-load-{int(time.time())}-{uuid4().hex[:6]}" + + ok = 0 + failures = [] + + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [ + executor.submit( + create_project, + args.base_url, + args.token, + boundary, + prefix, + index, + args.retries, + ) + for index in range(args.count) + ] + + for completed, future in enumerate(as_completed(futures), 1): + success, slug, status, text = future.result() + if success: + ok += 1 + else: + failures.append((slug, status, text[:500])) + + if completed % 50 == 0: + print( + f"completed={completed} created={ok} failed={len(failures)}", + flush=True, + ) + + print( + json.dumps( + { + "prefix": prefix, + "attempted": args.count, + "created": ok, + "failed": len(failures), + "failures": failures[:20], + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() From 534c63e66491fdf1bf03c07fe21a8d8215349a21 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:10:00 +0100 Subject: [PATCH 05/26] parallelize remove tasks --- apps/labrinth/src/env.rs | 1 + .../src/search/backend/typesense/mod.rs | 121 +++++++++++++++--- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index 69c5e8e7e6..20c6472316 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -165,6 +165,7 @@ vars! { TYPESENSE_URL: String = "http://localhost:8108"; TYPESENSE_API_KEY: String = "modrinth"; TYPESENSE_INDEX_PREFIX: String = "labrinth"; + TYPESENSE_DELETE_BATCH_SIZE: usize = 10_000usize; // storage STORAGE_BACKEND: crate::file_hosting::FileHostKind = crate::file_hosting::FileHostKind::Local; diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index 42beb818a5..0f39656db2 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -32,6 +32,7 @@ pub struct TypesenseConfig { pub index_prefix: String, pub meta_namespace: String, pub index_chunk_size: i64, + pub delete_batch_size: usize, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -160,6 +161,7 @@ impl TypesenseConfig { index_prefix: ENV.TYPESENSE_INDEX_PREFIX.clone(), meta_namespace: meta_namespace.unwrap_or_default(), index_chunk_size: ENV.SEARCH_INDEX_CHUNK_SIZE, + delete_batch_size: ENV.TYPESENSE_DELETE_BATCH_SIZE, } } @@ -323,12 +325,13 @@ impl TypesenseClient { &self, collection: &str, filter_by: &str, + batch_size: usize, ) -> Result<()> { let resp = self .request( Method::DELETE, &format!( - "/collections/{collection}/documents?filter_by={}&batch_size=1000", + "/collections/{collection}/documents?filter_by={}&batch_size={batch_size}", urlencoding::encode(filter_by) ), ) @@ -721,6 +724,24 @@ impl Typesense { self.client.upsert_alias(alias, &name).await?; Ok(()) } + + async fn delete_documents_by_filter_if_exists( + &self, + collection: &str, + filter: &str, + ) -> Result<()> { + if self.client.collection_exists(collection).await? { + self.client + .delete_documents_by_filter( + collection, + filter, + self.config.delete_batch_size, + ) + .await?; + } + + Ok(()) + } } #[async_trait] @@ -1050,20 +1071,53 @@ impl SearchBackend for Typesense { self.config.get_next_collection_name(&alias, false); debug!("Got shadow current {shadow_current:?}"); - for collection in - live.into_iter().chain([shadow_alt, shadow_current]) - { - debug!("Working on collection {collection:?}"); - if self.client.collection_exists(&collection).await? { + let delete_live = async { + if let Some(collection) = live.as_deref() { + debug!("Working on collection {collection:?}"); debug!( filter_len = filter.len(), "Collection exists, deleting by filter" ); - self.client - .delete_documents_by_filter(&collection, &filter) - .await?; + self.delete_documents_by_filter_if_exists( + collection, &filter, + ) + .await?; } - } + + Ok::<(), eyre::Report>(()) + }; + let delete_shadow_alt = async { + if live.as_deref() != Some(shadow_alt.as_str()) { + debug!("Working on collection {shadow_alt:?}"); + self.delete_documents_by_filter_if_exists( + &shadow_alt, + &filter, + ) + .await?; + } + + Ok::<(), eyre::Report>(()) + }; + let delete_shadow_current = async { + if live.as_deref() != Some(shadow_current.as_str()) { + debug!("Working on collection {shadow_current:?}"); + self.delete_documents_by_filter_if_exists( + &shadow_current, + &filter, + ) + .await?; + } + + Ok::<(), eyre::Report>(()) + }; + let (live_result, shadow_alt_result, shadow_current_result) = tokio::join!( + delete_live, + delete_shadow_alt, + delete_shadow_current + ); + live_result?; + shadow_alt_result?; + shadow_current_result?; } debug!("Done"); @@ -1092,15 +1146,46 @@ impl SearchBackend for Typesense { let shadow_current = self.config.get_next_collection_name(&alias, false); - for collection in - live.into_iter().chain([shadow_alt, shadow_current]) - { - if self.client.collection_exists(&collection).await? { - self.client - .delete_documents_by_filter(&collection, &filter) - .await?; + let delete_live = async { + if let Some(collection) = live.as_deref() { + self.delete_documents_by_filter_if_exists( + collection, &filter, + ) + .await?; } - } + + Ok::<(), eyre::Report>(()) + }; + let delete_shadow_alt = async { + if live.as_deref() != Some(shadow_alt.as_str()) { + self.delete_documents_by_filter_if_exists( + &shadow_alt, + &filter, + ) + .await?; + } + + Ok::<(), eyre::Report>(()) + }; + let delete_shadow_current = async { + if live.as_deref() != Some(shadow_current.as_str()) { + self.delete_documents_by_filter_if_exists( + &shadow_current, + &filter, + ) + .await?; + } + + Ok::<(), eyre::Report>(()) + }; + let (live_result, shadow_alt_result, shadow_current_result) = tokio::join!( + delete_live, + delete_shadow_alt, + delete_shadow_current + ); + live_result?; + shadow_alt_result?; + shadow_current_result?; } Ok(()) } From 3b771d7e8348674beb0bacae30c79e94deb83f0b Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:39:25 +0100 Subject: [PATCH 06/26] delete/upsert project/version messages --- apps/labrinth/src/queue/server_ping.rs | 20 ++- .../routes/internal/moderation/tech_review.rs | 1 + apps/labrinth/src/routes/v3/organizations.rs | 3 + .../src/routes/v3/project_creation.rs | 2 + .../src/routes/v3/project_creation/new.rs | 1 + apps/labrinth/src/routes/v3/projects.rs | 58 ++++---- .../src/routes/v3/version_creation.rs | 4 +- apps/labrinth/src/routes/v3/versions.rs | 2 + apps/labrinth/src/search/incremental.rs | 130 +++++++++++++--- .../src/search/incremental/consume.rs | 140 ++++++++++++++++-- apps/labrinth/src/search/indexing.rs | 2 +- 11 files changed, 298 insertions(+), 65 deletions(-) diff --git a/apps/labrinth/src/queue/server_ping.rs b/apps/labrinth/src/queue/server_ping.rs index 5b6305961b..95359701d2 100644 --- a/apps/labrinth/src/queue/server_ping.rs +++ b/apps/labrinth/src/queue/server_ping.rs @@ -1,9 +1,9 @@ use crate::database::DBProject; -use crate::database::models::DBProjectId; +use crate::database::models::{DBProjectId, DBVersionId}; use crate::database::redis::RedisPool; use crate::env::ENV; use crate::models::exp; -use crate::models::ids::ProjectId; +use crate::models::ids::{ProjectId, VersionId}; use crate::models::projects::ProjectStatus; use crate::search::incremental::IncrementalSearchQueue; use crate::{database::PgPool, util::error::Context}; @@ -175,14 +175,26 @@ impl ServerPingQueue { } if updated_project { + let version_ids = sqlx::query_scalar!( + "SELECT id FROM versions WHERE mod_id = $1", + DBProjectId::from(*project_id) as DBProjectId, + ) + .fetch_all(&self.db) + .await + .wrap_err("failed to fetch project version IDs")? + .into_iter() + .map(|version_id| VersionId::from(DBVersionId(version_id))) + .collect::>(); + let clear_cache = DBProject::clear_cache( (*project_id).into(), None, None, &self.redis, ); - let queue_search = - self.incremental_search_queue.push(*project_id); + let queue_search = self + .incremental_search_queue + .push(*project_id, version_ids); let (clear_cache_result, _) = join(clear_cache, queue_search).await; diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review.rs b/apps/labrinth/src/routes/internal/moderation/tech_review.rs index cfa609a98e..1ccdbafb49 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review.rs @@ -1142,6 +1142,7 @@ async fn submit_report( if verdict == DelphiVerdict::Unsafe { crate::routes::v3::projects::clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_id, diff --git a/apps/labrinth/src/routes/v3/organizations.rs b/apps/labrinth/src/routes/v3/organizations.rs index b59e74cefd..b27b3ce4dc 100644 --- a/apps/labrinth/src/routes/v3/organizations.rs +++ b/apps/labrinth/src/routes/v3/organizations.rs @@ -802,6 +802,7 @@ pub async fn organization_delete( for project_id in organization_project_ids { super::projects::clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_id, @@ -969,6 +970,7 @@ pub async fn organization_projects_add( ) .await?; super::projects::clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, @@ -1159,6 +1161,7 @@ pub async fn organization_projects_remove( ) .await?; super::projects::clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, diff --git a/apps/labrinth/src/routes/v3/project_creation.rs b/apps/labrinth/src/routes/v3/project_creation.rs index 1088302d2f..5e239543e7 100644 --- a/apps/labrinth/src/routes/v3/project_creation.rs +++ b/apps/labrinth/src/routes/v3/project_creation.rs @@ -349,6 +349,7 @@ pub async fn project_create_internal( } else { transaction.commit().await?; super::projects::clear_project_cache_and_queue_search( + &client, &redis, &search_state, project_id.into(), @@ -407,6 +408,7 @@ pub async fn project_create_with_id( } else { transaction.commit().await?; super::projects::clear_project_cache_and_queue_search( + &client, &redis, &search_state, project_id.into(), diff --git a/apps/labrinth/src/routes/v3/project_creation/new.rs b/apps/labrinth/src/routes/v3/project_creation/new.rs index 877ad9a70d..d3d025a1ee 100644 --- a/apps/labrinth/src/routes/v3/project_creation/new.rs +++ b/apps/labrinth/src/routes/v3/project_creation/new.rs @@ -342,6 +342,7 @@ pub async fn create( .wrap_internal_err("failed to commit transaction")?; super::super::projects::clear_project_cache_and_queue_search( + &db, &redis, &search_state, project_id.into(), diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index 902f62122f..07298a9c25 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -75,6 +75,7 @@ pub fn utoipa_config( } pub async fn clear_project_cache_and_queue_search( + pool: &PgPool, redis: &RedisPool, search_state: &SearchState, project_id: db_ids::DBProjectId, @@ -88,7 +89,21 @@ pub async fn clear_project_cache_and_queue_search( redis, ) .await?; - search_state.queue.push(project_id.into()).await; + let version_ids = sqlx::query_scalar!( + "SELECT id FROM versions WHERE mod_id = $1", + project_id as db_ids::DBProjectId, + ) + .fetch_all(pool) + .await + .wrap_internal_err("failed to fetch project version IDs")? + .into_iter() + .map(|version_id| VersionId::from(db_ids::DBVersionId(version_id))) + .collect::>(); + + search_state + .queue + .push(project_id.into(), version_ids) + .await; Ok(()) } @@ -1135,6 +1150,7 @@ pub async fn project_edit_internal( transaction.commit().await?; clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, @@ -1149,16 +1165,9 @@ pub async fn project_edit_internal( new_project.status.map(|status| status.is_searchable()), ) { search_state - .backend - .remove_documents( - &project_item - .versions - .into_iter() - .map(|x| x.into()) - .collect::>(), - ) - .await - .wrap_internal_err("failed to remove documents")?; + .queue + .push_project_removal(project_item.inner.id.into()) + .await; } Ok(HttpResponse::NoContent().body("")) @@ -1640,6 +1649,7 @@ pub async fn projects_edit( for (project_id, slug) in changed_projects { clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_id, @@ -1862,6 +1872,7 @@ pub async fn project_icon_edit_internal( transaction.commit().await?; clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, @@ -1977,6 +1988,7 @@ pub async fn delete_project_icon_internal( transaction.commit().await?; clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, @@ -2164,6 +2176,7 @@ pub async fn add_gallery_item_internal( transaction.commit().await?; clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, @@ -2370,6 +2383,7 @@ pub async fn edit_gallery_item_internal( transaction.commit().await?; clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, @@ -2510,6 +2524,7 @@ pub async fn delete_gallery_item_internal( transaction.commit().await?; clear_project_cache_and_queue_search( + &pool, &redis, &search_state, project_item.inner.id, @@ -2652,27 +2667,18 @@ pub async fn project_delete_internal( .await .wrap_internal_err("failed to commit transaction")?; - search_state - .backend - .remove_documents( - &project - .versions - .into_iter() - .map(|x| x.into()) - .collect::>(), - ) - .await - .wrap_internal_err("failed to remove project version documents")?; - if result.is_some() { - clear_project_cache_and_queue_search( - &redis, - &search_state, + db_models::DBProject::clear_cache( project.inner.id, project.inner.slug, None, + &redis, ) .await?; + search_state + .queue + .push_project_removal(project.inner.id.into()) + .await; Ok(()) } else { Err(ApiError::NotFound) diff --git a/apps/labrinth/src/routes/v3/version_creation.rs b/apps/labrinth/src/routes/v3/version_creation.rs index 9749efee3c..f9746a9a63 100644 --- a/apps/labrinth/src/routes/v3/version_creation.rs +++ b/apps/labrinth/src/routes/v3/version_creation.rs @@ -148,6 +148,7 @@ pub async fn version_create( } else if let Ok((_, project_id)) = &result { transaction.commit().await?; super::projects::clear_project_cache_and_queue_search( + &client, &redis, &search_state, *project_id, @@ -565,7 +566,7 @@ pub async fn upload_file_to_version( let result = upload_file_to_version_inner( req, &mut payload, - client, + client.clone(), &mut transaction, redis.clone(), &**file_host, @@ -591,6 +592,7 @@ pub async fn upload_file_to_version( } else if let Ok((_, project_id)) = &result { transaction.commit().await?; super::projects::clear_project_cache_and_queue_search( + &client, &redis, &search_state, *project_id, diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index 0e322cde51..155e754d67 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -762,6 +762,7 @@ pub async fn version_edit_helper( database::models::DBVersion::clear_cache(&version_item, &redis) .await?; super::projects::clear_project_cache_and_queue_search( + &pool, &redis, &search_state, version_item.inner.project_id, @@ -1098,6 +1099,7 @@ pub async fn version_delete( transaction.commit().await?; super::projects::clear_project_cache_and_queue_search( + &pool, &redis, &search_state, version.inner.project_id, diff --git a/apps/labrinth/src/search/incremental.rs b/apps/labrinth/src/search/incremental.rs index 88a9f1d6a2..1c57a57e9f 100644 --- a/apps/labrinth/src/search/incremental.rs +++ b/apps/labrinth/src/search/incremental.rs @@ -1,13 +1,18 @@ pub mod consume; -use std::{collections::HashSet, mem, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + mem, + sync::Arc, + time::Duration, +}; use rdkafka::{producer::FutureRecord, util::Timeout}; use serde::Serialize; use tokio::sync::Mutex; use crate::{ - models::ids::ProjectId, + models::ids::{ProjectId, VersionId}, util::kafka::{KAFKA_OPERATION_INTERVAL, KafkaClientState, KafkaEvent}, }; @@ -17,20 +22,36 @@ const QUEUE_FLUSH_INTERVAL: Duration = Duration::from_secs(10); #[derive(Clone)] pub struct IncrementalSearchQueue { - project_ids: Arc>>, + operations: Arc>, kafka_client: actix_web::web::Data, } impl IncrementalSearchQueue { pub fn new(kafka_client: actix_web::web::Data) -> Self { Self { - project_ids: Arc::new(Mutex::new(HashSet::new())), + operations: Arc::new(Mutex::new( + PendingSearchIndexOperations::default(), + )), kafka_client, } } - pub async fn push(&self, project_id: ProjectId) { - self.project_ids.lock().await.insert(project_id); + pub async fn push( + &self, + project_id: ProjectId, + version_ids: impl IntoIterator, + ) { + self.operations + .lock() + .await + .push_project_change(project_id, version_ids); + } + + pub async fn push_project_removal(&self, project_id: ProjectId) { + self.operations + .lock() + .await + .push_project_removal(project_id); } pub async fn run(self) { @@ -46,20 +67,20 @@ impl IncrementalSearchQueue { } pub async fn drain(&self) -> eyre::Result<()> { - let project_ids = { - let mut project_ids = self.project_ids.lock().await; - mem::take(&mut *project_ids) + let operations = { + let mut operations = self.operations.lock().await; + mem::take(&mut *operations) }; - if project_ids.is_empty() { + if operations.is_empty() { return Ok(()); } - let mut project_ids = project_ids.into_iter(); - while let Some(project_id) = project_ids.next() { + let mut operations = operations.into_events().into_iter(); + while let Some(operation) = operations.next() { let event = KafkaEvent::new( SEARCH_PROJECT_INDEX_QUEUE_TOPIC, - SearchProjectIndexQueueEventData { project_id }, + operation.clone(), ); let event_id = event.event_metadata.event_id; let key = event_id.to_string(); @@ -74,9 +95,11 @@ impl IncrementalSearchQueue { .send(record, Timeout::After(KAFKA_OPERATION_INTERVAL)) .await { - let mut queued_project_ids = self.project_ids.lock().await; - queued_project_ids.insert(project_id); - queued_project_ids.extend(project_ids); + let mut queued_operations = self.operations.lock().await; + queued_operations.push_event(operation); + for operation in operations { + queued_operations.push_event(operation); + } return Err(err.into()); } @@ -86,7 +109,76 @@ impl IncrementalSearchQueue { } } -#[derive(Debug, Serialize)] -pub struct SearchProjectIndexQueueEventData { - pub project_id: ProjectId, +#[derive(Default)] +struct PendingSearchIndexOperations { + changed_projects: HashMap>, + removed_project_ids: HashSet, +} + +impl PendingSearchIndexOperations { + fn is_empty(&self) -> bool { + self.changed_projects.is_empty() && self.removed_project_ids.is_empty() + } + + fn push_project_change( + &mut self, + project_id: ProjectId, + version_ids: impl IntoIterator, + ) { + if !self.removed_project_ids.contains(&project_id) { + self.changed_projects + .entry(project_id) + .or_default() + .extend(version_ids); + } + } + + fn push_project_removal(&mut self, project_id: ProjectId) { + self.changed_projects.remove(&project_id); + self.removed_project_ids.insert(project_id); + } + + fn push_event(&mut self, event: SearchProjectIndexQueueEventData) { + match event { + SearchProjectIndexQueueEventData::ProjectChange { + project_id, + version_ids, + } => self.push_project_change(project_id, version_ids), + SearchProjectIndexQueueEventData::ProjectRemoval { project_id } => { + self.push_project_removal(project_id) + } + } + } + + fn into_events(self) -> Vec { + let mut events = Vec::with_capacity( + self.changed_projects.len() + self.removed_project_ids.len(), + ); + + events.extend(self.removed_project_ids.into_iter().map(|project_id| { + SearchProjectIndexQueueEventData::ProjectRemoval { project_id } + })); + events.extend(self.changed_projects.into_iter().map( + |(project_id, version_ids)| { + SearchProjectIndexQueueEventData::ProjectChange { + project_id, + version_ids: version_ids.into_iter().collect(), + } + }, + )); + + events + } +} + +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum SearchProjectIndexQueueEventData { + ProjectChange { + project_id: ProjectId, + version_ids: Vec, + }, + ProjectRemoval { + project_id: ProjectId, + }, } diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 58e8a6456a..9713185c8a 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -15,7 +15,7 @@ use tracing::{Instrument, info, info_span}; use crate::{ database::{PgPool, redis::RedisPool}, env::ENV, - models::ids::ProjectId, + models::ids::{ProjectId, VersionId}, search::{ SearchBackend, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC, indexing::index_project_documents, @@ -128,8 +128,9 @@ async fn consume_batch( ) -> eyre::Result<()> { let start = Instant::now(); - let mut project_ids = Vec::new(); - let mut seen_project_ids = HashSet::new(); + let mut project_ids_to_change = HashSet::new(); + let mut project_ids_to_remove = HashSet::new(); + let mut version_ids_to_change = HashSet::new(); let mut messages_to_commit = Vec::new(); for message in messages { @@ -164,24 +165,94 @@ async fn consume_batch( } }; - if seen_project_ids.insert(event.project_id) { - project_ids.push(event.project_id); + match event.into_data() { + SearchProjectIndexQueueEventData::ProjectChange { + project_id, + version_ids, + } => { + project_ids_to_change.insert(project_id); + version_ids_to_change.extend(version_ids); + } + SearchProjectIndexQueueEventData::ProjectRemoval { project_id } => { + project_ids_to_remove.insert(project_id); + } } messages_to_commit.push(message); } + project_ids_to_change + .retain(|project_id| !project_ids_to_remove.contains(project_id)); + + let project_ids_to_change = + project_ids_to_change.into_iter().collect::>(); + let project_ids_to_remove = + project_ids_to_remove.into_iter().collect::>(); + let version_ids_to_change = + version_ids_to_change.into_iter().collect::>(); + info!( kafka.message_count = messages_to_commit.len(), - "Read all Kafka messages in {:.2?}, found {} projects to reindex", + "Read all Kafka messages in {:.2?}, found {} projects to change, {} versions to change, and {} projects to remove", start.elapsed(), - project_ids.len(), + project_ids_to_change.len(), + version_ids_to_change.len(), + project_ids_to_remove.len(), ); let start = Instant::now(); - if !project_ids.is_empty() { - reindex_projects(ro_pool, redis_pool, search_backend, &project_ids) + if !project_ids_to_remove.is_empty() { + let operation_start = Instant::now(); + info!( + project_count = project_ids_to_remove.len(), + "Removing project documents" + ); + search_backend + .remove_project_documents(&project_ids_to_remove) + .await + .wrap_err("failed to remove project documents")?; + info!( + project_count = project_ids_to_remove.len(), + "Removed project documents in {:.2?}", + operation_start.elapsed() + ); + } + + if !version_ids_to_change.is_empty() { + let operation_start = Instant::now(); + info!( + version_count = version_ids_to_change.len(), + "Removing changed version documents" + ); + search_backend + .remove_documents(&version_ids_to_change) .await - .wrap_err("failed to reindex project batch")?; + .wrap_err("failed to remove changed version documents")?; + info!( + version_count = version_ids_to_change.len(), + "Removed changed version documents in {:.2?}", + operation_start.elapsed() + ); + } + + if !project_ids_to_change.is_empty() { + let operation_start = Instant::now(); + info!( + project_count = project_ids_to_change.len(), + "Indexing changed projects" + ); + index_changed_projects( + ro_pool, + redis_pool, + search_backend, + &project_ids_to_change, + ) + .await + .wrap_err("failed to index changed project batch")?; + info!( + project_count = project_ids_to_change.len(), + "Indexed changed projects in {:.2?}", + operation_start.elapsed() + ); } for message in messages_to_commit { @@ -191,8 +262,9 @@ async fn consume_batch( } info!( - "Reindexed {} projects in {:.2?}", - project_ids.len(), + "Changed {} projects and removed {} projects in {:.2?}", + project_ids_to_change.len(), + project_ids_to_remove.len(), start.elapsed() ); @@ -218,6 +290,18 @@ pub async fn reindex_projects( search_backend.remove_project_documents(project_ids).await?; info!("Creating project documents"); + index_changed_projects(ro_pool, redis_pool, search_backend, project_ids) + .await?; + + Ok(()) +} + +async fn index_changed_projects( + ro_pool: &PgPool, + redis_pool: &RedisPool, + search_backend: &dyn SearchBackend, + project_ids: &[ProjectId], +) -> eyre::Result<()> { let documents = index_project_documents(ro_pool, redis_pool, project_ids) .instrument(info_span!("index", batch_size = project_ids.len())) .await @@ -236,6 +320,34 @@ pub async fn reindex_projects( } #[derive(Debug, Deserialize)] -struct SearchProjectIndexQueueEvent { - project_id: ProjectId, +#[serde(untagged)] +enum SearchProjectIndexQueueEvent { + Current(SearchProjectIndexQueueEventData), + Legacy { project_id: ProjectId }, +} + +impl SearchProjectIndexQueueEvent { + fn into_data(self) -> SearchProjectIndexQueueEventData { + match self { + Self::Current(data) => data, + Self::Legacy { project_id } => { + SearchProjectIndexQueueEventData::ProjectChange { + project_id, + version_ids: Vec::new(), + } + } + } + } +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum SearchProjectIndexQueueEventData { + ProjectChange { + project_id: ProjectId, + version_ids: Vec, + }, + ProjectRemoval { + project_id: ProjectId, + }, } diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 77e334cee0..75e564722d 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -7,7 +7,7 @@ use itertools::Itertools; use regex::Regex; use std::collections::HashMap; use std::sync::LazyLock; -use tracing::{debug, info, warn}; +use tracing::{info, warn}; use crate::database::PgPool; use crate::database::models::loader_fields::{ From 876af7de3829b7ab713b28d4b1de34c7636adb86 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:46:10 +0100 Subject: [PATCH 07/26] prepare --- ...51f87424ee39aac49cae5575c0d3313fb7f24.json | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json diff --git a/apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json b/apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json new file mode 100644 index 0000000000..59ab74bde2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM versions WHERE mod_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24" +} From d78883c130f227ea2c155faaaba10c69186f83d6 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:13:21 +0100 Subject: [PATCH 08/26] more logging --- apps/labrinth/src/search/backend/typesense/mod.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index 0f39656db2..4ba421614f 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -1026,10 +1026,15 @@ impl SearchBackend for Typesense { let shadow_current = self.config.get_next_collection_name(&alias, false); + debug!( + "Inserting into alias {alias:?}, live {live:?}, shadow alt {shadow_alt:?}, shadow current {shadow_current:?}" + ); + for collection in live.into_iter().chain([shadow_alt, shadow_current]) { if self.client.collection_exists(&collection).await? { + debug!("Inserting into existing collection {collection:?}"); self.client .import_documents(&collection, jsonl.clone()) .await?; @@ -1037,6 +1042,7 @@ impl SearchBackend for Typesense { } } + debug!("Done importing"); Ok(()) } From 4b7905488b3bfccee2d9cdb40cb615cd6b7f3c94 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:15:33 +0100 Subject: [PATCH 09/26] log number of docs --- apps/labrinth/src/search/backend/typesense/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index 4ba421614f..a6b2d69f6c 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -1016,6 +1016,7 @@ impl SearchBackend for Typesense { return Ok(()); } + let num_documents = documents.len(); let jsonl = documents_to_jsonl(documents)?; for alias in [ self.config.get_alias_name("projects"), @@ -1027,7 +1028,12 @@ impl SearchBackend for Typesense { self.config.get_next_collection_name(&alias, false); debug!( - "Inserting into alias {alias:?}, live {live:?}, shadow alt {shadow_alt:?}, shadow current {shadow_current:?}" + ?alias, + ?live, + ?shadow_alt, + ?shadow_current, + num_documents, + "Inserting into alias", ); for collection in From 674862602291047f588fc1d0a5e8b31e5cbad969 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sat, 27 Jun 2026 14:27:51 +0100 Subject: [PATCH 10/26] try more targeted, homogenous version change ops --- apps/labrinth/src/routes/v3/projects.rs | 34 +++++-- .../src/routes/v3/version_creation.rs | 26 ++++-- apps/labrinth/src/routes/v3/versions.rs | 8 +- apps/labrinth/src/search/incremental.rs | 49 +++++++--- .../src/search/incremental/consume.rs | 93 ++++++++++++++++--- apps/labrinth/src/search/indexing.rs | 67 ++++++++++++- 6 files changed, 229 insertions(+), 48 deletions(-) diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index 07298a9c25..ea3d9cda3b 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -82,13 +82,6 @@ pub async fn clear_project_cache_and_queue_search( slug: Option, clear_dependencies: Option, ) -> Result<(), ApiError> { - db_models::DBProject::clear_cache( - project_id, - slug, - clear_dependencies, - redis, - ) - .await?; let version_ids = sqlx::query_scalar!( "SELECT id FROM versions WHERE mod_id = $1", project_id as db_ids::DBProjectId, @@ -100,6 +93,33 @@ pub async fn clear_project_cache_and_queue_search( .map(|version_id| VersionId::from(db_ids::DBVersionId(version_id))) .collect::>(); + clear_project_cache_and_queue_search_versions( + redis, + search_state, + project_id, + slug, + clear_dependencies, + version_ids, + ) + .await +} + +pub async fn clear_project_cache_and_queue_search_versions( + redis: &RedisPool, + search_state: &SearchState, + project_id: db_ids::DBProjectId, + slug: Option, + clear_dependencies: Option, + version_ids: impl IntoIterator, +) -> Result<(), ApiError> { + db_models::DBProject::clear_cache( + project_id, + slug, + clear_dependencies, + redis, + ) + .await?; + search_state .queue .push(project_id.into(), version_ids) diff --git a/apps/labrinth/src/routes/v3/version_creation.rs b/apps/labrinth/src/routes/v3/version_creation.rs index f9746a9a63..a1a3424418 100644 --- a/apps/labrinth/src/routes/v3/version_creation.rs +++ b/apps/labrinth/src/routes/v3/version_creation.rs @@ -145,20 +145,20 @@ pub async fn version_create( if let Err(e) = rollback_result { return Err(e.into()); } - } else if let Ok((_, project_id)) = &result { + } else if let Ok((_, project_id, version_id)) = &result { transaction.commit().await?; - super::projects::clear_project_cache_and_queue_search( - &client, + super::projects::clear_project_cache_and_queue_search_versions( &redis, &search_state, *project_id, None, Some(true), + [VersionId::from(*version_id)], ) .await?; } - result.map(|(response, _)| response) + result.map(|(response, _, _)| response) } #[allow(clippy::too_many_arguments)] @@ -173,7 +173,8 @@ async fn version_create_inner( session_queue: &AuthQueue, moderation_queue: &AutomatedModerationQueue, http: &reqwest::Client, -) -> Result<(HttpResponse, models::DBProjectId), CreateError> { +) -> Result<(HttpResponse, models::DBProjectId, models::DBVersionId), CreateError> +{ let mut initial_version_data = None; let mut version_builder = None; let mut selected_loaders = None; @@ -544,7 +545,11 @@ async fn version_create_inner( moderation_queue.projects.insert(project_id.into()); } - Ok((HttpResponse::Ok().json(response), project_id)) + Ok(( + HttpResponse::Ok().json(response), + project_id, + models::DBVersionId::from(version_id), + )) } pub async fn upload_file_to_version( @@ -561,7 +566,8 @@ pub async fn upload_file_to_version( let mut transaction = client.begin().await?; let mut uploaded_files = Vec::new(); - let version_id = models::DBVersionId::from(url_data.into_inner().0); + let version_id = url_data.into_inner().0; + let db_version_id = models::DBVersionId::from(version_id); let result = upload_file_to_version_inner( req, @@ -571,7 +577,7 @@ pub async fn upload_file_to_version( redis.clone(), &**file_host, &mut uploaded_files, - version_id, + db_version_id, &session_queue, &http, ) @@ -591,13 +597,13 @@ pub async fn upload_file_to_version( } } else if let Ok((_, project_id)) = &result { transaction.commit().await?; - super::projects::clear_project_cache_and_queue_search( - &client, + super::projects::clear_project_cache_and_queue_search_versions( &redis, &search_state, *project_id, None, Some(true), + [version_id], ) .await?; } diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index 155e754d67..c9b8cda4b3 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -761,13 +761,13 @@ pub async fn version_edit_helper( transaction.commit().await?; database::models::DBVersion::clear_cache(&version_item, &redis) .await?; - super::projects::clear_project_cache_and_queue_search( - &pool, + super::projects::clear_project_cache_and_queue_search_versions( &redis, &search_state, version_item.inner.project_id, None, Some(true), + [VersionId::from(version_item.inner.id)], ) .await?; Ok(HttpResponse::NoContent().body("")) @@ -1098,13 +1098,13 @@ pub async fn version_delete( transaction.commit().await?; - super::projects::clear_project_cache_and_queue_search( - &pool, + super::projects::clear_project_cache_and_queue_search_versions( &redis, &search_state, version.inner.project_id, None, Some(true), + [VersionId::from(version.inner.id)], ) .await?; search_backend diff --git a/apps/labrinth/src/search/incremental.rs b/apps/labrinth/src/search/incremental.rs index 1c57a57e9f..b5ba4f0d0a 100644 --- a/apps/labrinth/src/search/incremental.rs +++ b/apps/labrinth/src/search/incremental.rs @@ -111,13 +111,16 @@ impl IncrementalSearchQueue { #[derive(Default)] struct PendingSearchIndexOperations { - changed_projects: HashMap>, + changed_project_ids: HashSet, + changed_project_versions: HashMap>, removed_project_ids: HashSet, } impl PendingSearchIndexOperations { fn is_empty(&self) -> bool { - self.changed_projects.is_empty() && self.removed_project_ids.is_empty() + self.changed_project_ids.is_empty() + && self.changed_project_versions.is_empty() + && self.removed_project_ids.is_empty() } fn push_project_change( @@ -126,24 +129,38 @@ impl PendingSearchIndexOperations { version_ids: impl IntoIterator, ) { if !self.removed_project_ids.contains(&project_id) { - self.changed_projects - .entry(project_id) - .or_default() - .extend(version_ids); + let version_ids = version_ids.into_iter().collect::>(); + if version_ids.is_empty() { + self.changed_project_versions.remove(&project_id); + self.changed_project_ids.insert(project_id); + } else if !self.changed_project_ids.contains(&project_id) { + self.changed_project_versions + .entry(project_id) + .or_default() + .extend(version_ids); + } } } fn push_project_removal(&mut self, project_id: ProjectId) { - self.changed_projects.remove(&project_id); + self.changed_project_ids.remove(&project_id); + self.changed_project_versions.remove(&project_id); self.removed_project_ids.insert(project_id); } fn push_event(&mut self, event: SearchProjectIndexQueueEventData) { match event { - SearchProjectIndexQueueEventData::ProjectChange { + SearchProjectIndexQueueEventData::ProjectChange { project_id } => { + self.push_project_change(project_id, []) + } + SearchProjectIndexQueueEventData::ProjectVersionChange { project_id, version_ids, - } => self.push_project_change(project_id, version_ids), + } => { + if !version_ids.is_empty() { + self.push_project_change(project_id, version_ids) + } + } SearchProjectIndexQueueEventData::ProjectRemoval { project_id } => { self.push_project_removal(project_id) } @@ -152,15 +169,20 @@ impl PendingSearchIndexOperations { fn into_events(self) -> Vec { let mut events = Vec::with_capacity( - self.changed_projects.len() + self.removed_project_ids.len(), + self.changed_project_ids.len() + + self.changed_project_versions.len() + + self.removed_project_ids.len(), ); events.extend(self.removed_project_ids.into_iter().map(|project_id| { SearchProjectIndexQueueEventData::ProjectRemoval { project_id } })); - events.extend(self.changed_projects.into_iter().map( + events.extend(self.changed_project_ids.into_iter().map(|project_id| { + SearchProjectIndexQueueEventData::ProjectChange { project_id } + })); + events.extend(self.changed_project_versions.into_iter().map( |(project_id, version_ids)| { - SearchProjectIndexQueueEventData::ProjectChange { + SearchProjectIndexQueueEventData::ProjectVersionChange { project_id, version_ids: version_ids.into_iter().collect(), } @@ -176,6 +198,9 @@ impl PendingSearchIndexOperations { pub enum SearchProjectIndexQueueEventData { ProjectChange { project_id: ProjectId, + }, + ProjectVersionChange { + project_id: ProjectId, version_ids: Vec, }, ProjectRemoval { diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 9713185c8a..5128712bc8 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -17,8 +17,9 @@ use crate::{ env::ENV, models::ids::{ProjectId, VersionId}, search::{ - SearchBackend, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC, - indexing::index_project_documents, + SearchBackend, + incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC, + indexing::{index_project_documents, index_project_version_documents}, }, util::kafka::{ INCREMENTAL_INDEX_SEARCH_TASK, KAFKA_OPERATION_INTERVAL, @@ -129,6 +130,7 @@ async fn consume_batch( let start = Instant::now(); let mut project_ids_to_change = HashSet::new(); + let mut project_ids_with_version_changes = HashSet::new(); let mut project_ids_to_remove = HashSet::new(); let mut version_ids_to_change = HashSet::new(); let mut messages_to_commit = Vec::new(); @@ -166,12 +168,17 @@ async fn consume_batch( }; match event.into_data() { - SearchProjectIndexQueueEventData::ProjectChange { + SearchProjectIndexQueueEventData::ProjectChange { project_id } => { + project_ids_to_change.insert(project_id); + } + SearchProjectIndexQueueEventData::ProjectVersionChange { project_id, version_ids, } => { - project_ids_to_change.insert(project_id); - version_ids_to_change.extend(version_ids); + if !version_ids.is_empty() { + project_ids_with_version_changes.insert(project_id); + version_ids_to_change.extend(version_ids); + } } SearchProjectIndexQueueEventData::ProjectRemoval { project_id } => { project_ids_to_remove.insert(project_id); @@ -182,9 +189,14 @@ async fn consume_batch( project_ids_to_change .retain(|project_id| !project_ids_to_remove.contains(project_id)); + project_ids_with_version_changes + .retain(|project_id| !project_ids_to_remove.contains(project_id)); let project_ids_to_change = project_ids_to_change.into_iter().collect::>(); + let project_ids_with_version_changes = project_ids_with_version_changes + .into_iter() + .collect::>(); let project_ids_to_remove = project_ids_to_remove.into_iter().collect::>(); let version_ids_to_change = @@ -192,9 +204,10 @@ async fn consume_batch( info!( kafka.message_count = messages_to_commit.len(), - "Read all Kafka messages in {:.2?}, found {} projects to change, {} versions to change, and {} projects to remove", + "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with version changes, {} versions to change, and {} projects to remove", start.elapsed(), project_ids_to_change.len(), + project_ids_with_version_changes.len(), version_ids_to_change.len(), project_ids_to_remove.len(), ); @@ -221,7 +234,7 @@ async fn consume_batch( let operation_start = Instant::now(); info!( version_count = version_ids_to_change.len(), - "Removing changed version documents" + "Removing changed version documents", ); search_backend .remove_documents(&version_ids_to_change) @@ -234,6 +247,30 @@ async fn consume_batch( ); } + if !project_ids_with_version_changes.is_empty() { + let operation_start = Instant::now(); + info!( + project_count = project_ids_with_version_changes.len(), + version_count = version_ids_to_change.len(), + "Indexing changed project versions" + ); + index_changed_project_versions( + ro_pool, + redis_pool, + search_backend, + &project_ids_with_version_changes, + &version_ids_to_change, + ) + .await + .wrap_err("failed to index changed project version batch")?; + info!( + project_count = project_ids_with_version_changes.len(), + version_count = version_ids_to_change.len(), + "Indexed changed project versions in {:.2?}", + operation_start.elapsed() + ); + } + if !project_ids_to_change.is_empty() { let operation_start = Instant::now(); info!( @@ -319,6 +356,40 @@ async fn index_changed_projects( Ok(()) } +async fn index_changed_project_versions( + ro_pool: &PgPool, + redis_pool: &RedisPool, + search_backend: &dyn SearchBackend, + project_ids: &[ProjectId], + version_ids: &[VersionId], +) -> eyre::Result<()> { + let documents = index_project_version_documents( + ro_pool, + redis_pool, + project_ids, + version_ids, + ) + .instrument(info_span!( + "index", + batch_size = project_ids.len(), + version_count = version_ids.len() + )) + .await + .wrap_err_with(|| { + format!( + "failed to build search documents for {} projects and {} versions", + project_ids.len(), + version_ids.len() + ) + })?; + + info!("Fetched all project version documents, indexing into backend"); + + search_backend.index_documents(&documents).await?; + + Ok(()) +} + #[derive(Debug, Deserialize)] #[serde(untagged)] enum SearchProjectIndexQueueEvent { @@ -331,10 +402,7 @@ impl SearchProjectIndexQueueEvent { match self { Self::Current(data) => data, Self::Legacy { project_id } => { - SearchProjectIndexQueueEventData::ProjectChange { - project_id, - version_ids: Vec::new(), - } + SearchProjectIndexQueueEventData::ProjectChange { project_id } } } } @@ -345,6 +413,9 @@ impl SearchProjectIndexQueueEvent { enum SearchProjectIndexQueueEventData { ProjectChange { project_id: ProjectId, + }, + ProjectVersionChange { + project_id: ProjectId, version_ids: Vec, }, ProjectRemoval { diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 75e564722d..4ebe510abd 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -5,7 +5,7 @@ use futures::TryStreamExt; use heck::ToKebabCase; use itertools::Itertools; use regex::Regex; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::LazyLock; use tracing::{info, warn}; @@ -20,7 +20,7 @@ use crate::database::models::{ }; use crate::database::redis::RedisPool; use crate::models::exp; -use crate::models::ids::ProjectId; +use crate::models::ids::{ProjectId, VersionId}; use crate::models::projects::{DependencyType, from_duplicate_version_fields}; use crate::models::v2::projects::LegacyProject; use crate::routes::v2_reroute; @@ -114,7 +114,8 @@ pub async fn index_local( return Ok((vec![], i64::MAX)); }; - let uploads = build_search_documents(pool, redis, db_projects).await?; + let uploads = + build_search_documents(pool, redis, db_projects, None).await?; Ok((uploads, *largest)) } @@ -163,13 +164,65 @@ pub async fn index_project_documents( info!("Fetched partial projects"); - build_search_documents(pool, redis, db_projects).await + build_search_documents(pool, redis, db_projects, None).await +} + +pub async fn index_project_version_documents( + pool: &PgPool, + redis: &RedisPool, + project_ids: &[ProjectId], + version_ids: &[VersionId], +) -> eyre::Result> { + let searchable_statuses = searchable_statuses(); + let project_ids = project_ids + .iter() + .map(|project_id| DBProjectId::from(*project_id).0) + .collect::>(); + let version_ids = version_ids + .iter() + .map(|version_id| DBVersionId::from(*version_id)) + .collect::>(); + + let db_projects = sqlx::query!( + r#" + SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, + m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color, + m.components AS "components: sqlx::types::Json" + FROM mods m + WHERE m.status = ANY($1) AND m.id = ANY($2) + GROUP BY m.id + ORDER BY m.id ASC; + "#, + &searchable_statuses, + &project_ids, + ) + .fetch(pool) + .map_ok(|m| PartialProject { + id: DBProjectId(m.id), + name: m.name, + summary: m.summary, + downloads: m.downloads, + follows: m.follows, + icon_url: m.icon_url, + updated: m.updated, + approved: m.approved.unwrap_or(m.published), + slug: m.slug, + color: m.color, + license: m.license, + components: m.components.0, + }) + .try_collect::>() + .await + .wrap_err("failed to fetch project")?; + + build_search_documents(pool, redis, db_projects, Some(&version_ids)).await } async fn build_search_documents( pool: &PgPool, redis: &RedisPool, db_projects: Vec, + version_ids_to_index: Option<&HashSet>, ) -> eyre::Result> { let searchable_statuses = searchable_statuses(); let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); @@ -501,6 +554,12 @@ async fn build_search_documents( .collect::>(); for version in versions { + if let Some(version_ids_to_index) = version_ids_to_index + && !version_ids_to_index.contains(&version.id) + { + continue; + } + let version_fields = VersionField::from_query_json( version.version_fields, &loader_fields, From e804ab0f79ce718517ffe97f82eec125ee755405 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sat, 27 Jun 2026 14:35:02 +0100 Subject: [PATCH 11/26] ensure only necessary fields are serialized into typesense --- apps/labrinth/src/search/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 9507aafc2e..d88a8ff656 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -232,16 +232,22 @@ impl FromStr for SearchBackendKind { } } +/// Nullable fields in Typesense-bound documents should use +/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of +/// serialized as `null`. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct UploadSearchProject { pub version_id: String, pub project_id: String, // pub project_types: Vec, + #[serde(skip_serializing_if = "Option::is_none")] pub slug: Option, pub author: String, pub author_id: String, + #[serde(skip_serializing_if = "Option::is_none")] pub organization: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub organization_id: Option, pub indexed_author: String, pub name: String, @@ -252,9 +258,11 @@ pub struct UploadSearchProject { pub follows: i32, pub downloads: i32, pub log_downloads: f64, + #[serde(skip_serializing_if = "Option::is_none")] pub icon_url: Option, pub license: String, pub gallery: Vec, + #[serde(skip_serializing_if = "Option::is_none")] pub featured_gallery: Option, /// RFC 3339 formatted creation date of the project pub date_created: DateTime, @@ -267,6 +275,7 @@ pub struct UploadSearchProject { /// Unix timestamp of the publication date of the version pub version_published_timestamp: i64, pub open_source: bool, + #[serde(skip_serializing_if = "Option::is_none")] pub color: Option, #[serde(default)] pub dependency_project_ids: Vec, @@ -285,12 +294,17 @@ pub struct UploadSearchProject { pub loader_fields: HashMap>, } +/// Nullable fields in Typesense-bound documents should use +/// `skip_serializing_if = "Option::is_none"` so they are omitted instead of +/// serialized as `null`. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct SearchProjectDependency { pub project_id: String, pub dependency_type: DependencyType, pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] pub slug: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub icon_url: Option, } From 9f17c29ee4359e1b232c958543d127428d805485 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 29 Jun 2026 19:48:13 +0100 Subject: [PATCH 12/26] disable index background task --- apps/labrinth/src/lib.rs | 24 -- scripts/import-typesense-jsonl.py | 517 ++++++++++++++++++++++++++++++ scripts/query-typesense.py | 105 ++++++ 3 files changed, 622 insertions(+), 24 deletions(-) create mode 100755 scripts/import-typesense-jsonl.py create mode 100644 scripts/query-typesense.py diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 20e09e8c1d..dd4339715c 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -145,30 +145,6 @@ pub fn app_setup( )); if enable_background_tasks { - // The interval in seconds at which the local database is indexed - // for searching. Defaults to 1 hour if unset. - let local_index_interval = - Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL); - let pool_ref = pool.clone(); - let redis_pool_ref = redis_pool.clone(); - let search_backend_ref = search_backend.clone(); - scheduler.run(local_index_interval, move || { - let pool_ref = pool_ref.clone(); - let redis_pool_ref = redis_pool_ref.clone(); - let search_backend = search_backend_ref.clone(); - async move { - if let Err(err) = background_task::index_search( - pool_ref, - redis_pool_ref, - search_backend, - ) - .await - { - warn!("Failed to index search: {err:?}"); - } - } - }); - // Changes statuses of scheduled projects/versions let pool_ref = pool.clone(); // TODO: Clear cache when these are run diff --git a/scripts/import-typesense-jsonl.py b/scripts/import-typesense-jsonl.py new file mode 100755 index 0000000000..90487ca4ec --- /dev/null +++ b/scripts/import-typesense-jsonl.py @@ -0,0 +1,517 @@ +#!/usr/bin/env python3 +""" +Import a JSONL Typesense export into local Labrinth Typesense collections. + +By default this imports tmp/export.jsonl into the Labrinth project search alias, +resolving the alias to its current backing collection first. + +Usage: + python3 scripts/import-typesense-jsonl.py + python3 scripts/import-typesense-jsonl.py tmp/export.jsonl --continue + python3 scripts/import-typesense-jsonl.py tmp/export.jsonl --alias labrinth_projects + python3 scripts/import-typesense-jsonl.py --collection labrinth_projects__alt +""" + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +class TypesenseError(Exception): + pass + + +def default_aliases(): + prefix = os.environ.get("TYPESENSE_INDEX_PREFIX", "labrinth") + return [f"{prefix}_projects"] + + +def request_typesense(base_url, api_key, method, path, timeout, body=None, content_type=None): + url = f"{base_url.rstrip('/')}{path}" + request = urllib.request.Request(url, data=body, method=method) + request.add_header("X-TYPESENSE-API-KEY", api_key) + if content_type: + request.add_header("Content-Type", content_type) + + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + except urllib.error.HTTPError as error: + error_body = error.read().decode("utf-8", errors="replace") + raise TypesenseError(f"{method} {path} failed ({error.code}): {error_body}") from error + except urllib.error.URLError as error: + raise TypesenseError(f"{method} {path} failed: {error.reason}") from error + + +def get_json_or_none(base_url, api_key, path, timeout): + try: + body = request_typesense(base_url, api_key, "GET", path, timeout) + except TypesenseError as error: + if " failed (404):" in str(error): + return None + raise + return json.loads(body.decode("utf-8")) + + +def request_json(base_url, api_key, method, path, timeout, body=None): + encoded_body = None + if body is not None: + encoded_body = json.dumps(body).encode("utf-8") + + response = request_typesense( + base_url, + api_key, + method, + path, + timeout, + body=encoded_body, + content_type="application/json" if body is not None else None, + ) + if not response: + return None + return json.loads(response.decode("utf-8")) + + +def resolve_alias_or_collection(base_url, api_key, name, timeout): + escaped = urllib.parse.quote(name, safe="") + alias = get_json_or_none(base_url, api_key, f"/aliases/{escaped}", timeout) + if alias is not None: + collection_name = alias.get("collection_name") + if not collection_name: + raise TypesenseError(f"Alias {name} did not include collection_name") + return collection_name + + collection = get_json_or_none(base_url, api_key, f"/collections/{escaped}", timeout) + if collection is None: + raise TypesenseError(f"No Typesense alias or collection exists for {name}") + return name + + +def collection_document_count(base_url, api_key, collection, timeout): + escaped = urllib.parse.quote(collection, safe="") + body = get_json_or_none(base_url, api_key, f"/collections/{escaped}", timeout) + if body is None: + raise TypesenseError(f"No Typesense collection exists for {collection}") + return int(body.get("num_documents", 0)) + + +def collection_schema_exists(base_url, api_key, collection, timeout): + escaped = urllib.parse.quote(collection, safe="") + return get_json_or_none(base_url, api_key, f"/collections/{escaped}", timeout) is not None + + +def collection_schema_for_create(name, source): + schema = { + "name": name, + "fields": source["fields"], + "default_sorting_field": source.get("default_sorting_field"), + "enable_nested_fields": source.get("enable_nested_fields", False), + "token_separators": source.get("token_separators", []), + "symbols_to_index": source.get("symbols_to_index", []), + } + return {key: value for key, value in schema.items() if value is not None} + + +def sibling_collection_name(collection): + if collection.endswith("__current"): + return f"{collection[:-len('__current')]}__alt" + if collection.endswith("__alt"): + return f"{collection[:-len('__alt')]}__current" + return None + + +def create_collection_from_source(base_url, api_key, collection, source_collection, timeout): + escaped_source = urllib.parse.quote(source_collection, safe="") + source = get_json_or_none(base_url, api_key, f"/collections/{escaped_source}", timeout) + if source is None: + raise TypesenseError( + f"Cannot create {collection}: source collection {source_collection} does not exist" + ) + + request_json( + base_url, + api_key, + "POST", + "/collections", + timeout, + body=collection_schema_for_create(collection, source), + ) + print(f"Created {collection} from schema of {source_collection}") + + +def ensure_collection_exists(collection, args): + if collection_schema_exists(args.typesense_url, args.api_key, collection, args.timeout): + return + + if not args.create_missing_collection: + raise TypesenseError(f"No Typesense collection exists for {collection}") + + source_collection = args.create_from_collection or sibling_collection_name(collection) + if not source_collection: + raise TypesenseError( + f"No Typesense collection exists for {collection}; pass --create-from-collection" + ) + + create_collection_from_source( + args.typesense_url, + args.api_key, + collection, + source_collection, + args.timeout, + ) + + +def typesense_health_ok(base_url, api_key, timeout): + try: + body = request_typesense(base_url, api_key, "GET", "/health", timeout) + except TypesenseError as error: + if " failed (503):" in str(error): + return False + raise + + return json.loads(body.decode("utf-8")).get("ok") is True + + +def wait_for_typesense_drain(collection, expected_documents, args): + started_at = time.monotonic() + last_reported_at = 0.0 + + while True: + document_count = collection_document_count( + args.typesense_url, args.api_key, collection, args.timeout + ) + healthy = typesense_health_ok(args.typesense_url, args.api_key, args.timeout) + + if document_count >= expected_documents and healthy: + elapsed = time.monotonic() - started_at + print( + f"{collection}: Typesense reports {document_count} document(s) " + f"and healthy after flush wait ({elapsed:.1f}s)" + ) + return + + elapsed = time.monotonic() - started_at + if elapsed > args.flush_timeout: + raise TypesenseError( + f"Timed out waiting for Typesense to flush {collection}: " + f"{document_count}/{expected_documents} document(s), " + f"health ok={healthy}" + ) + + if elapsed - last_reported_at >= args.flush_report_interval: + print( + f"{collection}: waiting for Typesense flush; " + f"{document_count}/{expected_documents} document(s), " + f"health ok={healthy} ({elapsed:.1f}s)" + ) + last_reported_at = elapsed + + time.sleep(args.flush_poll_interval) + + +def unique(values): + result = [] + seen = set() + for value in values: + if value in seen: + continue + seen.add(value) + result.append(value) + return result + + +def iter_jsonl_batches(path, batch_lines, max_batch_bytes, skip_documents=0): + batch = [] + batch_bytes = 0 + start_line = None + end_line = None + skipped = 0 + + with open(path, "rb") as export_file: + for line_number, line in enumerate(export_file, start=1): + if not line.strip(): + continue + + if skipped < skip_documents: + skipped += 1 + continue + + line_bytes = len(line) + should_flush = batch and ( + len(batch) >= batch_lines or batch_bytes + line_bytes > max_batch_bytes + ) + if should_flush: + yield start_line, end_line, len(batch), b"".join(batch) + batch = [] + batch_bytes = 0 + start_line = None + + if start_line is None: + start_line = line_number + end_line = line_number + batch.append(line) + batch_bytes += line_bytes + + if batch: + yield start_line, end_line, len(batch), b"".join(batch) + + +def count_jsonl_batches(path, batch_lines, max_batch_bytes, skip_documents=0): + batch_count = 0 + batch_lines_count = 0 + batch_bytes = 0 + skipped = 0 + + with open(path, "rb") as export_file: + for line in export_file: + if not line.strip(): + continue + + if skipped < skip_documents: + skipped += 1 + continue + + line_bytes = len(line) + if batch_lines_count and ( + batch_lines_count >= batch_lines or batch_bytes + line_bytes > max_batch_bytes + ): + batch_count += 1 + batch_lines_count = 0 + batch_bytes = 0 + + batch_lines_count += 1 + batch_bytes += line_bytes + + if batch_lines_count: + batch_count += 1 + + return batch_count + + +def parse_import_response(body, failures_to_print): + imported = 0 + failed = 0 + failures = [] + text = body.decode("utf-8", errors="replace") + + for line in text.splitlines(): + if not line: + continue + try: + result = json.loads(line) + except json.JSONDecodeError: + failed += 1 + if len(failures) < failures_to_print: + failures.append(line) + continue + + if result.get("success") is True: + imported += 1 + else: + failed += 1 + if len(failures) < failures_to_print: + failures.append(json.dumps(result, sort_keys=True)) + + if imported == 0 and failed == 0 and text.strip(): + failed = 1 + failures.append(text.strip()) + + return imported, failed, failures + + +def import_batch(base_url, api_key, collection, args, body): + query = urllib.parse.urlencode( + { + "action": args.action, + "dirty_values": args.dirty_values, + } + ) + escaped = urllib.parse.quote(collection, safe="") + if not body.endswith(b"\n"): + body += b"\n" + + return request_typesense( + base_url, + api_key, + "POST", + f"/collections/{escaped}/documents/import?{query}", + args.timeout, + body=body, + content_type="text/plain", + ) + + +def import_file(collection, args): + imported_total = 0 + batch_count = 0 + skip_documents = 0 + next_flush_at = args.flush_interval + started_at = time.monotonic() + + ensure_collection_exists(collection, args) + + if args.continue_import: + skip_documents = collection_document_count( + args.typesense_url, args.api_key, collection, args.timeout + ) + print(f"{collection}: continuing after {skip_documents} existing document(s)") + + total_batches = count_jsonl_batches( + args.file, args.batch_lines, args.max_batch_bytes, skip_documents + ) + if total_batches == 0: + print(f"{collection}: no remaining documents to import") + return 0 + + for start_line, end_line, document_count, body in iter_jsonl_batches( + args.file, args.batch_lines, args.max_batch_bytes, skip_documents + ): + batch_count += 1 + response = import_batch(args.typesense_url, args.api_key, collection, args, body) + imported, failed, failures = parse_import_response(response, args.failures_to_print) + + if failed: + print( + f"Typesense reported {failed} failed document(s) for " + f"{collection}, input lines {start_line}-{end_line}.", + file=sys.stderr, + ) + for failure in failures: + print(failure, file=sys.stderr) + raise TypesenseError(f"Import into {collection} failed") + + if imported != document_count: + raise TypesenseError( + f"Typesense acknowledged {imported} document(s) for " + f"{collection}, but the batch contained {document_count}" + ) + + imported_total += imported + elapsed = time.monotonic() - started_at + percent = batch_count / total_batches * 100 + print( + f"{collection}: batch {batch_count}/{total_batches} ({percent:.1f}%) " + f"imported {imported} document(s) " + f"from lines {start_line}-{end_line}; {imported_total} total ({elapsed:.1f}s)" + ) + + if args.flush_interval and imported_total >= next_flush_at: + expected_documents = skip_documents + imported_total + wait_for_typesense_drain(collection, expected_documents, args) + while imported_total >= next_flush_at: + next_flush_at += args.flush_interval + + elapsed = time.monotonic() - started_at + print(f"{collection}: imported {imported_total} documents in {batch_count} batches ({elapsed:.1f}s)") + return imported_total + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Import a JSONL export into Typesense in payload-safe batches." + ) + parser.add_argument("jsonl_file", nargs="?", help="JSONL export to import") + parser.add_argument("--file", help="JSONL export to import") + parser.add_argument( + "--typesense-url", + default=os.environ.get("TYPESENSE_URL", "http://localhost:8108"), + help="Typesense URL", + ) + parser.add_argument( + "--api-key", + default=os.environ.get("TYPESENSE_API_KEY", "modrinth"), + help="Typesense API key", + ) + parser.add_argument( + "--alias", + action="append", + dest="aliases", + help="Typesense alias or collection name to import into. Repeat for multiple targets.", + ) + parser.add_argument( + "--collection", + action="append", + dest="collections", + help="Exact Typesense collection name. Repeat for multiple targets.", + ) + parser.add_argument("--action", default="create", choices=["create", "update", "upsert", "emplace"]) + parser.add_argument("--dirty-values", default="coerce_or_drop") + parser.add_argument("--batch-lines", type=int, default=5_000) + parser.add_argument("--max-batch-bytes", type=int, default=4 * 1024 * 1024) + parser.add_argument("--flush-interval", type=int, default=10_000) + parser.add_argument("--flush-timeout", type=float, default=900.0) + parser.add_argument("--flush-poll-interval", type=float, default=2.0) + parser.add_argument("--flush-report-interval", type=float, default=10.0) + parser.add_argument( + "--create-missing-collection", + action=argparse.BooleanOptionalAction, + default=True, + help="Create a missing __current/__alt collection by cloning its sibling schema.", + ) + parser.add_argument( + "--create-from-collection", + help="Collection whose schema should be cloned when creating a missing target collection.", + ) + parser.add_argument("--failures-to-print", type=int, default=20) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument( + "--continue", + action="store_true", + dest="continue_import", + help="Resume by skipping the number of documents already in the target collection.", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + + if args.batch_lines <= 0: + raise TypesenseError("--batch-lines must be greater than zero") + if args.max_batch_bytes <= 0: + raise TypesenseError("--max-batch-bytes must be greater than zero") + if args.flush_interval < 0: + raise TypesenseError("--flush-interval must be greater than or equal to zero") + if args.flush_timeout <= 0: + raise TypesenseError("--flush-timeout must be greater than zero") + if args.flush_poll_interval <= 0: + raise TypesenseError("--flush-poll-interval must be greater than zero") + if args.flush_report_interval <= 0: + raise TypesenseError("--flush-report-interval must be greater than zero") + if args.aliases and args.collections: + raise TypesenseError("Use either --alias or --collection, not both") + if args.jsonl_file and args.file: + raise TypesenseError("Pass the JSONL path either positionally or with --file, not both") + args.file = args.jsonl_file or args.file or "tmp/export.jsonl" + if not os.path.isfile(args.file): + raise TypesenseError(f"File does not exist: {args.file}") + + if args.collections: + collections = unique(args.collections) + else: + aliases = unique(args.aliases or default_aliases()) + collections = [ + resolve_alias_or_collection(args.typesense_url, args.api_key, alias, args.timeout) + for alias in aliases + ] + collections = unique(collections) + + print(f"Importing {args.file} into {', '.join(collections)}") + for collection in collections: + import_file(collection, args) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("Interrupted.", file=sys.stderr) + sys.exit(130) + except TypesenseError as error: + print(error, file=sys.stderr) + sys.exit(1) diff --git a/scripts/query-typesense.py b/scripts/query-typesense.py new file mode 100644 index 0000000000..24f151bf75 --- /dev/null +++ b/scripts/query-typesense.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Query the local Labrinth Typesense collection directly. + +Usage: + python3 scripts/query-typesense.py sodium + python3 scripts/query-typesense.py foobarqux --per-page 10 +""" + +import argparse +import json +import os +import urllib.error +import urllib.request + + +def request_json(base_url, api_key, path, body, timeout): + request = urllib.request.Request( + f"{base_url.rstrip('/')}{path}", + data=json.dumps(body).encode("utf-8"), + method="POST", + headers={ + "X-TYPESENSE-API-KEY": api_key, + "Content-Type": "application/json", + }, + ) + + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.load(response) + except urllib.error.HTTPError as error: + error_body = error.read().decode("utf-8", errors="replace") + raise SystemExit(f"Typesense request failed ({error.code}): {error_body}") from error + except urllib.error.URLError as error: + raise SystemExit(f"Typesense request failed: {error.reason}") from error + + +def parse_args(): + parser = argparse.ArgumentParser(description="Query Typesense directly.") + parser.add_argument("query", help="Search query") + parser.add_argument( + "--typesense-url", + default=os.environ.get("TYPESENSE_URL", "http://localhost:8108"), + help="Typesense URL", + ) + parser.add_argument( + "--api-key", + default=os.environ.get("TYPESENSE_API_KEY", "modrinth"), + help="Typesense API key", + ) + parser.add_argument( + "--collection", + default="labrinth_projects__current", + help="Typesense collection to search", + ) + parser.add_argument("--per-page", type=int, default=5) + parser.add_argument("--timeout", type=float, default=30.0) + return parser.parse_args() + + +def main(): + args = parse_args() + payload = { + "searches": [ + { + "collection": args.collection, + "q": args.query, + "query_by": "name,summary,author,indexed_name,indexed_author", + "per_page": args.per_page, + "group_by": "project_id", + "group_limit": 1, + "facet_by": "project_id", + "max_facet_values": 0, + } + ] + } + + body = request_json( + args.typesense_url, args.api_key, "/multi_search", payload, args.timeout + ) + result = body["results"][0] + + print(f"query: {args.query}") + print(f"collection: {args.collection}") + print(f"found: {result.get('found', 0)}") + print(f"out_of: {result.get('out_of', 0)}") + print(f"search_time_ms: {result.get('search_time_ms', 0)}") + print() + + grouped_hits = result.get("grouped_hits", []) + if not grouped_hits: + print("no hits") + return + + for index, group in enumerate(grouped_hits, start=1): + hit = group["hits"][0] + document = hit["document"] + print(f"{index}. {document.get('name', '')} ({document.get('slug', '')})") + print(f" project_id: {document.get('project_id', '')}") + print(f" author: {document.get('author', '')}") + print(f" summary: {document.get('summary', '')}") + + +if __name__ == "__main__": + main() From 8c041d8a15c8c708effbc0f1f8a25dd66a23bc44 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:27:15 +0100 Subject: [PATCH 13/26] wip: script changes --- scripts/import-typesense-jsonl.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/import-typesense-jsonl.py b/scripts/import-typesense-jsonl.py index 90487ca4ec..93b2fa03c7 100755 --- a/scripts/import-typesense-jsonl.py +++ b/scripts/import-typesense-jsonl.py @@ -193,7 +193,7 @@ def wait_for_typesense_drain(collection, expected_documents, args): f"{collection}: Typesense reports {document_count} document(s) " f"and healthy after flush wait ({elapsed:.1f}s)" ) - return + return elapsed elapsed = time.monotonic() - started_at if elapsed > args.flush_timeout: @@ -352,6 +352,8 @@ def import_file(collection, args): skip_documents = 0 next_flush_at = args.flush_interval started_at = time.monotonic() + flush_batch_started_at = started_at + flush_batch_start_total = 0 ensure_collection_exists(collection, args) @@ -372,7 +374,9 @@ def import_file(collection, args): args.file, args.batch_lines, args.max_batch_bytes, skip_documents ): batch_count += 1 + batch_started_at = time.monotonic() response = import_batch(args.typesense_url, args.api_key, collection, args, body) + batch_elapsed = time.monotonic() - batch_started_at imported, failed, failures = parse_import_response(response, args.failures_to_print) if failed: @@ -397,14 +401,29 @@ def import_file(collection, args): print( f"{collection}: batch {batch_count}/{total_batches} ({percent:.1f}%) " f"imported {imported} document(s) " - f"from lines {start_line}-{end_line}; {imported_total} total ({elapsed:.1f}s)" + f"from lines {start_line}-{end_line}; {imported_total} total " + f"(batch {batch_elapsed:.1f}s, elapsed {elapsed:.1f}s)" ) if args.flush_interval and imported_total >= next_flush_at: expected_documents = skip_documents + imported_total - wait_for_typesense_drain(collection, expected_documents, args) + flush_batch_elapsed = time.monotonic() - flush_batch_started_at + flush_batch_documents = imported_total - flush_batch_start_total + print( + f"{collection}: flush checkpoint at {imported_total} imported document(s); " + f"imported {flush_batch_documents} document(s) since previous flush " + f"in {flush_batch_elapsed:.1f}s" + ) + flush_wait_elapsed = wait_for_typesense_drain(collection, expected_documents, args) + print( + f"{collection}: flush checkpoint complete; " + f"import batch {flush_batch_elapsed:.1f}s, " + f"flush wait {flush_wait_elapsed:.1f}s" + ) while imported_total >= next_flush_at: next_flush_at += args.flush_interval + flush_batch_started_at = time.monotonic() + flush_batch_start_total = imported_total elapsed = time.monotonic() - started_at print(f"{collection}: imported {imported_total} documents in {batch_count} batches ({elapsed:.1f}s)") From a27a4828ef7658dee681ff6b7a9c578fd836c5d4 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:26:17 +0100 Subject: [PATCH 14/26] don't facet by project id --- .../labrinth/src/search/backend/typesense/mod.rs | 13 +------------ scripts/.gitignore | 1 + .../create-dummy-projects.cpython-314.pyc | Bin 5414 -> 0 bytes 3 files changed, 2 insertions(+), 12 deletions(-) create mode 100644 scripts/.gitignore delete mode 100644 scripts/__pycache__/create-dummy-projects.cpython-314.pyc diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index a6b2d69f6c..4db90b621c 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -796,8 +796,6 @@ impl SearchBackend for Typesense { ("per_page", parsed.hits_per_page.to_string()), ("group_by", "project_id".to_string()), ("group_limit", "1".to_string()), - ("facet_by", "project_id".to_string()), - ("max_facet_values", "0".to_string()), ( "max_candidates", info.typesense_config.max_candidates.to_string(), @@ -853,16 +851,7 @@ impl SearchBackend for Typesense { .cloned() .unwrap_or(body); - let total_hits = body["facet_counts"] - .as_array() - .and_then(|facets| { - facets.iter().find(|facet| { - facet["field_name"].as_str() == Some("project_id") - }) - }) - .and_then(|facet| facet["stats"]["total_values"].as_u64()) - .unwrap_or_else(|| body["found"].as_u64().unwrap_or(0)) - as usize; + let total_hits = body["found"].as_u64().unwrap_or(0) as usize; let hits = body["grouped_hits"] .as_array() diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000000..bee8a64b79 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/scripts/__pycache__/create-dummy-projects.cpython-314.pyc b/scripts/__pycache__/create-dummy-projects.cpython-314.pyc deleted file mode 100644 index 5100c7d3103b7fd09b5ee18fc966255066873c37..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5414 zcma)AYit|G5#De7}6>&p%KQI z)JUVHG}>qxqb0}Klbpt#NkJ1lNTlnLW)&HzZ6XV`{TYYI9r!LnaX3da zddA*{`m5UlH_>kV_wk=%+F2i|yMj}S6paTmndGsnQY@=w6x|k8Be6_MPD+{-_Ypdi z%_idgF{&yDH?{B|k01RUsyq@wY={&|JK9T%REQ91kr7#u6Zu{wT0}v#LTeN4eY{MC zVMV0`Oof;PXxmVT66-|AV0BKW#hMVZA+a_@L+v#BuDx7YNJns4|_~HR_EUEV7chB*irEL`Lx@GqGqAYFMO6 zs+LGk_UhKSq{ftltR*t(+`-^fLiL(Uu0&Puq?DGFDCn!QI5}+yx){}BQ{F^6E?os7 zhA_2PuT7*AS|XZ^JS!OwLRlhUdbNyK&P%vcb< zg`i4u6gJ8zYL4{#bKkKGM>1(mN^Ab3302OhAUTsB!n>#Z@u(IZ_F_dG+K&6%*O-IT zvNYt4$}(&b#Y_4wshPB0unRfbE`VU#@3#x^K!_fl81hj%jpua+lqIV=E2U$Zxa4DW z{$eJZj)Nu@3w#BgG486QsiPLKCLvvGO!_}=Ap`?`y@YmYL1zU6|NP9_7aSClt^pQK2RcSiq3N-(p|V% zB5kwr66q+AxMIpqi-pR&SGFQs?Tw+I4He?^xx3cA`TF}bo4=aBR7Rw;yUZfio*yeC z!oiien*5~#Tei?t!(#!_!py)=4wUUIX?p}`Krt5A)xtw-*uaxe=i%ML`$QpHB)|$N znGF*rke$f~1} zts!f`zF`pzP5T5#B%Hq-aD0_RwCJSaPEY&6O-UMno$B)v>=c1cjm2Oss$y0Yu#P32is5rt_=hjvM7E5%ad(E+c(`Qu;r_3 zMZ4$_YfL@Kc95`j6Rz3=uCRfmAYV#YU^QQC!s%SKfP1GoXRwQKJdXho04E79P2nOk&-f5 zaKz|rj>&2h{sTIvNOCe7lVF;a6a`#wVg*d{xXu_O-I`5^DXqZcLOLre z(nR8_f?o+foQC5~#f8&`@7-=pL^VxH$(l~V@$rr-Yyx+NZKbQ4iVYaty5ZTqMp5v4 z(j67&9;xbFISNGl5L6YkmPaM4bGEb4Rya1-UKlR&p1k`W?<{oA?_AA4fdI&kl`Mfcu?0}EY6*P${)*{FQ|ih!J33zPH2&CBoCZqJ{;$2)E~UT`e)t&4nX ziFM?^H+!u}HI;Z<$?7TzwPg#kcpf3fLg(Ad4uR_a(t-x1j}?WcC1D`nUt%rSr)H)Kb z7qD@SUz91}`Fw>(6nDMr$6YTyRd5zg+}t)NF4?`yWSjalDF4qbz9TO39oBPrgnMVF zfcqoXBQ@4L0uB8;H59I0o?{1?J9|diV>I*bAOVB-XdAAbM&IQ=wvTzQ!|3m0fF1+E ztEC2dIEFo%h3Jw;--h$dXdt5y+6>KMOevD4_w;-n7X?xD+qszVYNW)tbg&w0zg8;0 z#y?K}AN-be{LBXaP2%zD-3d{UNzobx?_VilBYnXAz-q5L1kO(L9m)>`y5mnC~4 znjb$1IED+c0qll1lN`e|2x7D_32(U>oeClp?lvVt zwq!A~C7d(4!hVw;VoYi9{aMJf+-vrADqGQwxCQ^MVXrl*;a*ctV7J*@Yuk{x6>_)w z&EJ670LZv;vy+IR3$bbao%i{D=jrnz=MlvF0JB+}Z{mlGYo2~Kj>N&nK zIU+NW%&Jpo6)-Zu64e+k#dWUAuCo3@)wt8g)LZOaiJ>CEYQq zOlDI+4d5aEfgm1_z^p;jDY&367@U}>Dgz)&!8xB!O-WZ3>}z%FIxQ((n8Fw;1>lpt zm`Z2{NKl6G%3)j`!Ymdk4O`%}S^?7n)B;C^Z=|!3w@T@_&d6A~I+X-igU_#EN93yk zbW=GeZ5#&!o{BEW%w^>q7Q{={-#V%QXd^GUy-gUzp*7<7Z>bBT263Ro11f4<GQ%en<#d_au=Ue-KJNWzIH-{IL-;ETzhKs_%{MP#{mv@zH zj(o!>Y{PtTdFTGco%@&Bfq$?AZ+AjE4G=8;%B7bt-K}kVw8d(tAGncodp=vPL$vMs z$jrz++J&*ynVB=Vd2;5-5^XEdLW#EE@#&eZF~O2%;Ed6+qs!@h2VVu zZTiN7-7-*NCdy0NW@p8h_I~ksO-fHFGSG1NS4W^K zQv^YLjA}kaR^wmYhiJ=3Xyl)0>qEAVus!T)CE6bzCiW9lStj5$%;xS=jUN!qPq#jx J5WUUV;C~)fnq&X~ From 11978fd47956533b6d654e3610e24dc394a7dbad Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 9 Jul 2026 13:31:52 +0100 Subject: [PATCH 15/26] fix --- ...0326fa6d4b6212a8788503d5a53cef0f0be8bb981.json | 15 +++++++++++++++ apps/labrinth/src/search/mod.rs | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json diff --git a/apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json b/apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json new file mode 100644 index 0000000000..f05a642c48 --- /dev/null +++ b/apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE mods\n SET queued = NOW()\n WHERE id = $1 AND status = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981" +} diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 87a7e538bb..7b1db63299 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -242,7 +242,7 @@ pub struct UploadSearchProject { pub project_id: String, // pub project_types: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde(default)] pub all_project_types: Vec, pub slug: Option, pub author: String, From 95ddb489a72d3affd0582333d8087229e4a36db1 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:09:04 +0100 Subject: [PATCH 16/26] fix clippy --- apps/labrinth/src/search/incremental.rs | 25 +++++++++---------- .../src/search/incremental/consume.rs | 21 ++++++++-------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/apps/labrinth/src/search/incremental.rs b/apps/labrinth/src/search/incremental.rs index b5ba4f0d0a..96e8b7e096 100644 --- a/apps/labrinth/src/search/incremental.rs +++ b/apps/labrinth/src/search/incremental.rs @@ -150,10 +150,10 @@ impl PendingSearchIndexOperations { fn push_event(&mut self, event: SearchProjectIndexQueueEventData) { match event { - SearchProjectIndexQueueEventData::ProjectChange { project_id } => { + SearchProjectIndexQueueEventData::Change { project_id } => { self.push_project_change(project_id, []) } - SearchProjectIndexQueueEventData::ProjectVersionChange { + SearchProjectIndexQueueEventData::VersionChange { project_id, version_ids, } => { @@ -161,7 +161,7 @@ impl PendingSearchIndexOperations { self.push_project_change(project_id, version_ids) } } - SearchProjectIndexQueueEventData::ProjectRemoval { project_id } => { + SearchProjectIndexQueueEventData::Removal { project_id } => { self.push_project_removal(project_id) } } @@ -175,14 +175,14 @@ impl PendingSearchIndexOperations { ); events.extend(self.removed_project_ids.into_iter().map(|project_id| { - SearchProjectIndexQueueEventData::ProjectRemoval { project_id } + SearchProjectIndexQueueEventData::Removal { project_id } })); events.extend(self.changed_project_ids.into_iter().map(|project_id| { - SearchProjectIndexQueueEventData::ProjectChange { project_id } + SearchProjectIndexQueueEventData::Change { project_id } })); events.extend(self.changed_project_versions.into_iter().map( |(project_id, version_ids)| { - SearchProjectIndexQueueEventData::ProjectVersionChange { + SearchProjectIndexQueueEventData::VersionChange { project_id, version_ids: version_ids.into_iter().collect(), } @@ -196,14 +196,13 @@ impl PendingSearchIndexOperations { #[derive(Debug, Clone, Serialize)] #[serde(tag = "type", rename_all = "snake_case")] pub enum SearchProjectIndexQueueEventData { - ProjectChange { - project_id: ProjectId, - }, - ProjectVersionChange { + #[serde(rename = "project_change")] + Change { project_id: ProjectId }, + #[serde(rename = "project_version_change")] + VersionChange { project_id: ProjectId, version_ids: Vec, }, - ProjectRemoval { - project_id: ProjectId, - }, + #[serde(rename = "project_removal")] + Removal { project_id: ProjectId }, } diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 5128712bc8..2cfd315f24 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -168,10 +168,10 @@ async fn consume_batch( }; match event.into_data() { - SearchProjectIndexQueueEventData::ProjectChange { project_id } => { + SearchProjectIndexQueueEventData::Change { project_id } => { project_ids_to_change.insert(project_id); } - SearchProjectIndexQueueEventData::ProjectVersionChange { + SearchProjectIndexQueueEventData::VersionChange { project_id, version_ids, } => { @@ -180,7 +180,7 @@ async fn consume_batch( version_ids_to_change.extend(version_ids); } } - SearchProjectIndexQueueEventData::ProjectRemoval { project_id } => { + SearchProjectIndexQueueEventData::Removal { project_id } => { project_ids_to_remove.insert(project_id); } } @@ -402,7 +402,7 @@ impl SearchProjectIndexQueueEvent { match self { Self::Current(data) => data, Self::Legacy { project_id } => { - SearchProjectIndexQueueEventData::ProjectChange { project_id } + SearchProjectIndexQueueEventData::Change { project_id } } } } @@ -411,14 +411,13 @@ impl SearchProjectIndexQueueEvent { #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] enum SearchProjectIndexQueueEventData { - ProjectChange { - project_id: ProjectId, - }, - ProjectVersionChange { + #[serde(rename = "project_change")] + Change { project_id: ProjectId }, + #[serde(rename = "project_version_change")] + VersionChange { project_id: ProjectId, version_ids: Vec, }, - ProjectRemoval { - project_id: ProjectId, - }, + #[serde(rename = "project_removal")] + Removal { project_id: ProjectId }, } From 5ddae3801496d2065c9a5b63c7e264e1b737bcee Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:26:08 +0100 Subject: [PATCH 17/26] batch by document and dedup loaders --- apps/labrinth/.env.docker-compose | 1 + apps/labrinth/.env.local | 1 + apps/labrinth/src/env.rs | 1 + .../src/search/backend/typesense/mod.rs | 103 ++++++++++++------ apps/labrinth/src/search/indexing.rs | 4 +- 5 files changed, 73 insertions(+), 37 deletions(-) diff --git a/apps/labrinth/.env.docker-compose b/apps/labrinth/.env.docker-compose index 7e403e09fc..871cbd8133 100644 --- a/apps/labrinth/.env.docker-compose +++ b/apps/labrinth/.env.docker-compose @@ -30,6 +30,7 @@ SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000 TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth +TYPESENSE_IMPORT_BATCH_SIZE=5000 REDIS_URL=redis://labrinth-redis REDIS_MIN_CONNECTIONS=0 diff --git a/apps/labrinth/.env.local b/apps/labrinth/.env.local index 01fb7f915c..25ef3e5ab5 100644 --- a/apps/labrinth/.env.local +++ b/apps/labrinth/.env.local @@ -48,6 +48,7 @@ SEARCH_INCREMENTAL_INDEX_BATCH_MAX_SIZE=1000 TYPESENSE_URL=http://localhost:8108 TYPESENSE_API_KEY=modrinth TYPESENSE_INDEX_PREFIX=labrinth +TYPESENSE_IMPORT_BATCH_SIZE=5000 REDIS_URL=redis://localhost REDIS_MIN_CONNECTIONS=0 diff --git a/apps/labrinth/src/env.rs b/apps/labrinth/src/env.rs index 8de8e53c61..f6a4705cd6 100644 --- a/apps/labrinth/src/env.rs +++ b/apps/labrinth/src/env.rs @@ -165,6 +165,7 @@ vars! { TYPESENSE_URL: String = "http://localhost:8108"; TYPESENSE_API_KEY: String = "modrinth"; TYPESENSE_INDEX_PREFIX: String = "labrinth"; + TYPESENSE_IMPORT_BATCH_SIZE: usize = 5000usize; TYPESENSE_DELETE_BATCH_SIZE: usize = 10_000usize; // storage diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index c6c37a0b84..f34c730e26 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -32,6 +32,7 @@ pub struct TypesenseConfig { pub index_prefix: String, pub meta_namespace: String, pub index_chunk_size: i64, + pub import_batch_size: usize, pub delete_batch_size: usize, } @@ -161,6 +162,7 @@ impl TypesenseConfig { index_prefix: ENV.TYPESENSE_INDEX_PREFIX.clone(), meta_namespace: meta_namespace.unwrap_or_default(), index_chunk_size: ENV.SEARCH_INDEX_CHUNK_SIZE, + import_batch_size: ENV.TYPESENSE_IMPORT_BATCH_SIZE, delete_batch_size: ENV.TYPESENSE_DELETE_BATCH_SIZE, } } @@ -751,6 +753,58 @@ impl Typesense { Ok(()) } + + async fn import_document_batches( + &self, + collections: &[String], + documents: &[UploadSearchProject], + ) -> Result<()> { + let batch_size = self.config.import_batch_size.max(1); + + for batch in documents.chunks(batch_size) { + let jsonl = documents_to_jsonl(batch)?; + + for collection in collections { + info!( + collection, + document_count = batch.len(), + content_length_bytes = jsonl.len(), + "sending Typesense document import" + ); + self.client + .import_documents(collection, jsonl.clone()) + .await?; + } + } + + Ok(()) + } + + async fn existing_write_collections(&self) -> Result> { + let mut collections = Vec::new(); + + for alias in [ + self.config.get_alias_name("projects"), + self.config.get_alias_name("projects_filtered"), + ] { + let live = self.client.get_alias(&alias).await?; + let shadow_alt = self.config.get_next_collection_name(&alias, true); + let shadow_current = + self.config.get_next_collection_name(&alias, false); + + for collection in + live.into_iter().chain([shadow_alt, shadow_current]) + { + if !collections.contains(&collection) + && self.client.collection_exists(&collection).await? + { + collections.push(collection); + } + } + } + + Ok(collections) + } } #[async_trait] @@ -979,11 +1033,11 @@ impl SearchBackend for Typesense { total += uploads.len(); cursor = next_cursor; - let jsonl = documents_to_jsonl(&uploads)?; - self.client - .import_documents(&projects_next, jsonl.clone()) - .await?; - self.client.import_documents(&filtered_next, jsonl).await?; + self.import_document_batches( + &[projects_next.clone(), filtered_next.clone()], + &uploads, + ) + .await?; } info!("swapping aliases"); @@ -1014,37 +1068,14 @@ impl SearchBackend for Typesense { return Ok(()); } - let num_documents = documents.len(); - let jsonl = documents_to_jsonl(documents)?; - for alias in [ - self.config.get_alias_name("projects"), - self.config.get_alias_name("projects_filtered"), - ] { - let live = self.client.get_alias(&alias).await?; - let shadow_alt = self.config.get_next_collection_name(&alias, true); - let shadow_current = - self.config.get_next_collection_name(&alias, false); - - debug!( - ?alias, - ?live, - ?shadow_alt, - ?shadow_current, - num_documents, - "Inserting into alias", - ); - - for collection in - live.into_iter().chain([shadow_alt, shadow_current]) - { - if self.client.collection_exists(&collection).await? { - debug!("Inserting into existing collection {collection:?}"); - self.client - .import_documents(&collection, jsonl.clone()) - .await?; - } - } - } + let collections = self.existing_write_collections().await?; + debug!( + ?collections, + num_documents = documents.len(), + "Inserting into collections", + ); + self.import_document_batches(&collections, documents) + .await?; debug!("Done importing"); Ok(()) diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 213ad59354..503e34e002 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -548,10 +548,12 @@ async fn build_search_documents( from_duplicate_version_fields(aggregated_version_fields); // aggregated project loaders - let project_loaders = versions + let mut project_loaders = versions .iter() .flat_map(|x| x.loaders.clone()) .collect::>(); + project_loaders.sort(); + project_loaders.dedup(); // all valid project types across every version of the project, so that // filters can exclude projects that have *any* version of a given From 61990b05991157d8e58660a4ecd73059e996bd77 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:44:47 +0100 Subject: [PATCH 18/26] fix test --- apps/labrinth/src/routes/v3/projects.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index 638a545a36..df93ab527a 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -2834,6 +2834,11 @@ pub async fn project_delete_internal( &redis, ) .await?; + search_state + .backend + .remove_project_documents(&[project.inner.id.into()]) + .await + .wrap_internal_err("failed to remove project from search index")?; search_state .queue .push_project_removal(project.inner.id.into()) From 77deb033f7de02239dab33c1502db02c89b343ce Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:49:33 +0100 Subject: [PATCH 19/26] wip: projects/versions collections --- apps/labrinth/src/queue/server_ping.rs | 20 +- .../routes/internal/moderation/tech_review.rs | 1 - apps/labrinth/src/routes/v3/organizations.rs | 3 - .../src/routes/v3/project_creation.rs | 2 - .../src/routes/v3/project_creation/new.rs | 1 - apps/labrinth/src/routes/v3/projects.rs | 82 +- apps/labrinth/src/routes/v3/versions.rs | 13 +- apps/labrinth/src/search/backend/common.rs | 17 - apps/labrinth/src/search/backend/mod.rs | 4 +- .../src/search/backend/typesense/mod.rs | 866 +++++++++++++----- apps/labrinth/src/search/incremental.rs | 46 +- .../src/search/incremental/consume.rs | 45 +- apps/labrinth/src/search/indexing.rs | 331 ++++--- apps/labrinth/src/search/mod.rs | 37 +- scripts/convert-typesense-project-docs.py | 443 +++++++++ 15 files changed, 1446 insertions(+), 465 deletions(-) create mode 100755 scripts/convert-typesense-project-docs.py diff --git a/apps/labrinth/src/queue/server_ping.rs b/apps/labrinth/src/queue/server_ping.rs index 95359701d2..5b6305961b 100644 --- a/apps/labrinth/src/queue/server_ping.rs +++ b/apps/labrinth/src/queue/server_ping.rs @@ -1,9 +1,9 @@ use crate::database::DBProject; -use crate::database::models::{DBProjectId, DBVersionId}; +use crate::database::models::DBProjectId; use crate::database::redis::RedisPool; use crate::env::ENV; use crate::models::exp; -use crate::models::ids::{ProjectId, VersionId}; +use crate::models::ids::ProjectId; use crate::models::projects::ProjectStatus; use crate::search::incremental::IncrementalSearchQueue; use crate::{database::PgPool, util::error::Context}; @@ -175,26 +175,14 @@ impl ServerPingQueue { } if updated_project { - let version_ids = sqlx::query_scalar!( - "SELECT id FROM versions WHERE mod_id = $1", - DBProjectId::from(*project_id) as DBProjectId, - ) - .fetch_all(&self.db) - .await - .wrap_err("failed to fetch project version IDs")? - .into_iter() - .map(|version_id| VersionId::from(DBVersionId(version_id))) - .collect::>(); - let clear_cache = DBProject::clear_cache( (*project_id).into(), None, None, &self.redis, ); - let queue_search = self - .incremental_search_queue - .push(*project_id, version_ids); + let queue_search = + self.incremental_search_queue.push(*project_id); let (clear_cache_result, _) = join(clear_cache, queue_search).await; diff --git a/apps/labrinth/src/routes/internal/moderation/tech_review.rs b/apps/labrinth/src/routes/internal/moderation/tech_review.rs index 0e1df4ec89..745509e834 100644 --- a/apps/labrinth/src/routes/internal/moderation/tech_review.rs +++ b/apps/labrinth/src/routes/internal/moderation/tech_review.rs @@ -1161,7 +1161,6 @@ pub async fn submit_report( if verdict == DelphiVerdict::Unsafe { crate::routes::v3::projects::clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_id, diff --git a/apps/labrinth/src/routes/v3/organizations.rs b/apps/labrinth/src/routes/v3/organizations.rs index 971e1fe191..8b71d75912 100644 --- a/apps/labrinth/src/routes/v3/organizations.rs +++ b/apps/labrinth/src/routes/v3/organizations.rs @@ -810,7 +810,6 @@ pub async fn organization_delete( for project_id in organization_project_ids { super::projects::clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_id, @@ -980,7 +979,6 @@ pub async fn organization_projects_add( ) .await?; super::projects::clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_item.inner.id, @@ -1173,7 +1171,6 @@ pub async fn organization_projects_remove( ) .await?; super::projects::clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_item.inner.id, diff --git a/apps/labrinth/src/routes/v3/project_creation.rs b/apps/labrinth/src/routes/v3/project_creation.rs index 137ff5809f..3640c34014 100644 --- a/apps/labrinth/src/routes/v3/project_creation.rs +++ b/apps/labrinth/src/routes/v3/project_creation.rs @@ -358,7 +358,6 @@ pub async fn project_create_internal( } else { transaction.commit().await?; super::projects::clear_project_cache_and_queue_search( - &client, &redis, &search_state, project_id.into(), @@ -425,7 +424,6 @@ pub async fn project_create_with_id( } else { transaction.commit().await?; super::projects::clear_project_cache_and_queue_search( - &client, &redis, &search_state, project_id.into(), diff --git a/apps/labrinth/src/routes/v3/project_creation/new.rs b/apps/labrinth/src/routes/v3/project_creation/new.rs index 51f5a1f3e8..968f2ba308 100644 --- a/apps/labrinth/src/routes/v3/project_creation/new.rs +++ b/apps/labrinth/src/routes/v3/project_creation/new.rs @@ -345,7 +345,6 @@ pub async fn create( .wrap_internal_err("failed to commit transaction")?; super::super::projects::clear_project_cache_and_queue_search( - &db, &redis, &search_state, project_id.into(), diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index df93ab527a..43ced679ee 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -72,35 +72,42 @@ pub fn project_config(cfg: &mut actix_web::web::ServiceConfig) { } pub async fn clear_project_cache_and_queue_search( - pool: &PgPool, redis: &RedisPool, search_state: &SearchState, project_id: db_ids::DBProjectId, slug: Option, clear_dependencies: Option, ) -> Result<(), ApiError> { - let version_ids = sqlx::query_scalar!( - "SELECT id FROM versions WHERE mod_id = $1", - project_id as db_ids::DBProjectId, - ) - .fetch_all(pool) - .await - .wrap_internal_err("failed to fetch project version IDs")? - .into_iter() - .map(|version_id| VersionId::from(db_ids::DBVersionId(version_id))) - .collect::>(); - - clear_project_cache_and_queue_search_versions( + clear_project_cache_and_queue_search_inner( redis, search_state, project_id, slug, clear_dependencies, - version_ids, ) .await } +pub async fn clear_project_cache_and_queue_search_inner( + redis: &RedisPool, + search_state: &SearchState, + project_id: db_ids::DBProjectId, + slug: Option, + clear_dependencies: Option, +) -> Result<(), ApiError> { + db_models::DBProject::clear_cache( + project_id, + slug, + clear_dependencies, + redis, + ) + .await?; + + search_state.queue.push(project_id.into()).await; + + Ok(()) +} + pub async fn clear_project_cache_and_queue_search_versions( redis: &RedisPool, search_state: &SearchState, @@ -119,7 +126,7 @@ pub async fn clear_project_cache_and_queue_search_versions( search_state .queue - .push(project_id.into(), version_ids) + .push_versions(project_id.into(), version_ids) .await; Ok(()) @@ -1133,6 +1140,9 @@ pub async fn project_edit_internal( Ok(()) } + let reindex_version_project_types = + new_project.minecraft_java_server.is_some(); + update( &mut transaction, id, @@ -1202,15 +1212,26 @@ pub async fn project_edit_internal( transaction.commit().await?; - clear_project_cache_and_queue_search( - &pool, - &redis, - &search_state, - project_item.inner.id, - project_item.inner.slug, - None, - ) - .await?; + if reindex_version_project_types { + clear_project_cache_and_queue_search_versions( + &redis, + &search_state, + project_item.inner.id, + project_item.inner.slug, + None, + project_item.versions.iter().copied().map(VersionId::from), + ) + .await?; + } else { + clear_project_cache_and_queue_search( + &redis, + &search_state, + project_item.inner.id, + project_item.inner.slug, + None, + ) + .await?; + } // Remove no longer searchable projects from search index if let (true, Some(false)) = ( @@ -1764,7 +1785,6 @@ pub async fn projects_edit( for (project_id, slug) in changed_projects { clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_id, @@ -1996,7 +2016,6 @@ pub async fn project_icon_edit_internal( transaction.commit().await?; clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_item.inner.id, @@ -2116,7 +2135,6 @@ pub async fn delete_project_icon_internal( transaction.commit().await?; clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_item.inner.id, @@ -2317,7 +2335,6 @@ pub async fn add_gallery_item_internal( transaction.commit().await?; clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_item.inner.id, @@ -2536,7 +2553,6 @@ pub async fn edit_gallery_item_internal( transaction.commit().await?; clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_item.inner.id, @@ -2685,7 +2701,6 @@ pub async fn delete_gallery_item_internal( transaction.commit().await?; clear_project_cache_and_queue_search( - &pool, &redis, &search_state, project_item.inner.id, @@ -2834,6 +2849,13 @@ pub async fn project_delete_internal( &redis, ) .await?; + search_state + .backend + .remove_project_version_documents(&[project.inner.id.into()]) + .await + .wrap_internal_err( + "failed to remove project versions from search index", + )?; search_state .backend .remove_project_documents(&[project.inner.id.into()]) diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index 3cf7d80c01..f912956fd1 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -27,6 +27,7 @@ use crate::models::teams::ProjectPermissions; use crate::queue::file_scan::get_files_missing_attribution; use crate::queue::session::AuthQueue; use crate::routes::internal::delphi; +use crate::search::incremental::consume::reindex_project; use crate::search::{SearchBackend, SearchState}; use crate::util::error::Context; use crate::util::img; @@ -1257,9 +1258,17 @@ pub async fn version_delete( ) .await?; search_backend - .remove_documents(&[version.inner.id.into()]) + .remove_version_documents(&[version.inner.id.into()]) .await - .wrap_internal_err("failed to remove documents")?; + .wrap_internal_err("failed to remove version search document")?; + reindex_project( + &pool, + &redis, + search_backend.as_ref(), + version.inner.project_id.into(), + ) + .await + .wrap_internal_err("failed to reindex project")?; if result.is_some() { Ok(HttpResponse::NoContent().body("")) } else { diff --git a/apps/labrinth/src/search/backend/common.rs b/apps/labrinth/src/search/backend/common.rs index 5192a83dc3..9589275186 100644 --- a/apps/labrinth/src/search/backend/common.rs +++ b/apps/labrinth/src/search/backend/common.rs @@ -50,14 +50,7 @@ pub enum SearchIndex { MinecraftJavaServerPlayersOnline, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SearchIndexName { - Projects, - ProjectsFiltered, -} - pub struct SearchSort { - pub index_name: SearchIndexName, pub index: SearchIndex, } @@ -65,16 +58,12 @@ pub fn parse_search_index( index: &str, new_filters: Option<&str>, ) -> Result { - let projects_name = SearchIndexName::Projects; - let projects_filtered_name = SearchIndexName::ProjectsFiltered; - // TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly let is_server = new_filters .is_some_and(|f| f.contains("project_types = minecraft_java_server")); Ok(match index { "relevance" => SearchSort { - index_name: projects_name, index: if is_server { SearchIndex::MinecraftJavaServerVerifiedPlays2w } else { @@ -82,27 +71,21 @@ pub fn parse_search_index( }, }, "downloads" => SearchSort { - index_name: projects_filtered_name, index: SearchIndex::Downloads, }, "follows" => SearchSort { - index_name: projects_name, index: SearchIndex::Follows, }, "updated" | "date_modified" => SearchSort { - index_name: projects_name, index: SearchIndex::Updated, }, "newest" | "date_created" => SearchSort { - index_name: projects_name, index: SearchIndex::Newest, }, "minecraft_java_server.verified_plays_2w" => SearchSort { - index_name: projects_name, index: SearchIndex::MinecraftJavaServerVerifiedPlays2w, }, "minecraft_java_server.ping.data.players_online" => SearchSort { - index_name: projects_name, index: SearchIndex::MinecraftJavaServerPlayersOnline, }, i => return Err(ApiError::Request(eyre!("invalid index '{i}'"))), diff --git a/apps/labrinth/src/search/backend/mod.rs b/apps/labrinth/src/search/backend/mod.rs index 307cde1040..544cf2a557 100644 --- a/apps/labrinth/src/search/backend/mod.rs +++ b/apps/labrinth/src/search/backend/mod.rs @@ -2,7 +2,7 @@ mod common; pub mod typesense; pub use common::{ - ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort, - combined_search_filters, parse_search_index, parse_search_request, + ParsedSearchRequest, SearchIndex, SearchSort, combined_search_filters, + parse_search_index, parse_search_request, }; pub use typesense::{Typesense, TypesenseConfig}; diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index f34c730e26..8033006d60 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -3,11 +3,12 @@ use std::sync::LazyLock; use ariadne::ids::base62_impl::to_base62; use async_trait::async_trait; use eyre::{Result, eyre}; +use itertools::Itertools; use regex::Regex; use reqwest::Method; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; -use tracing::{debug, info, warn}; +use tracing::{debug, info}; use crate::database::PgPool; use crate::database::redis::RedisPool; @@ -15,13 +16,13 @@ use crate::env::ENV; use crate::models::ids::{ProjectId, VersionId}; use crate::routes::ApiError; use crate::search::backend::{ - SearchIndex, SearchIndexName, combined_search_filters, parse_search_index, + SearchIndex, combined_search_filters, parse_search_index, parse_search_request, }; use crate::search::indexing::index_local; use crate::search::{ ResultSearchProject, SearchBackend, SearchField, SearchRequest, - SearchResults, TasksCancelFilter, UploadSearchProject, + SearchResults, TasksCancelFilter, UploadSearchProject, UploadSearchVersion, }; use crate::util::error::Context; @@ -239,6 +240,22 @@ impl TypesenseClient { Ok(()) } + async fn delete_alias_if_exists(&self, alias: &str) -> Result<()> { + let resp = self + .request(Method::DELETE, &format!("/aliases/{alias}")) + .send() + .await + .wrap_err("failed to DELETE Typesense alias")?; + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + if !resp.status().is_success() { + let body = resp.json::().await.unwrap_or_default(); + return Err(eyre!("failed to delete alias `{alias}`: {body}")); + } + Ok(()) + } + async fn collection_exists(&self, name: &str) -> Result { let resp = self .request(Method::GET, &format!("/collections/{name}")) @@ -304,21 +321,35 @@ impl TypesenseClient { )); } // Typesense always returns HTTP 200; individual lines signal per-doc success. - let error_count = body + let failures = body .lines() - .filter(|l| !l.trim().is_empty()) - .filter(|l| { - serde_json::from_str::(l) - .ok() - .and_then(|v| v["success"].as_bool()) - .map(|ok| !ok) - .unwrap_or(false) + .filter(|line| !line.trim().is_empty()) + .filter_map(|line| match serde_json::from_str::(line) { + Ok(result) if result["success"].as_bool() == Some(true) => None, + Ok(result) => Some( + result["error"] + .as_str() + .unwrap_or( + "Typesense returned an unsuccessful import result", + ) + .to_string(), + ), + Err(err) => Some(format!( + "failed to parse Typesense import result: {err}" + )), }) - .count(); - if error_count > 0 { - warn!( - "{error_count} document(s) failed to import into `{collection}`" - ); + .collect::>(); + if !failures.is_empty() { + let failure_count = failures.len(); + let errors = failures + .into_iter() + .unique() + .take(10) + .collect::>() + .join("; "); + return Err(eyre!( + "{failure_count} document(s) failed to import into `{collection}`: {errors}" + )); } Ok(()) } @@ -368,6 +399,18 @@ pub struct TypesenseFieldSpec { } impl SearchField { + const fn is_version_field(self) -> bool { + matches!( + self, + Self::Categories + | Self::ProjectTypes + | Self::Environment + | Self::GameVersions + | Self::ClientSide + | Self::ServerSide + ) + } + pub const fn typesense_spec(self) -> TypesenseFieldSpec { match self { SearchField::Categories => TypesenseFieldSpec { @@ -569,7 +612,7 @@ impl Typesense { Self { config, client } } - fn collection_schema(name: &str) -> Value { + fn project_collection_schema(name: &str) -> Value { let mut fields = vec![ json!({"name": "summary", "type": "string", "facet": false}), json!({"name": "slug", "type": "string", "facet": false}), @@ -585,6 +628,7 @@ impl Typesense { json!({"name": "minecraft_java_server.is_online", "type": "bool", "sort": true, "optional": true}), json!({"name": "minecraft_java_server.ping.data.players_online", "type": "int32", "sort": true, "optional": true}), json!({"name": "dependencies", "type": "object[]", "optional": true}), + json!({"name": "project_categories", "type": "string[]", "facet": true, "optional": true}), ]; fields.extend(TYPESENSE_SEARCH_FIELDS.iter().cloned()); @@ -596,6 +640,50 @@ impl Typesense { }) } + fn version_collection_schema( + name: &str, + projects_collection: &str, + ) -> Value { + use strum::IntoEnumIterator; + + let mut fields = SearchField::iter() + .filter(|field| field.is_version_field()) + .map(|field| { + let spec = field.typesense_spec(); + json!({ + "name": spec.path, + "type": spec.ty, + "facet": spec.facet, + "optional": spec.optional, + "token_separators": spec.token_separators, + }) + }) + .collect::>(); + fields.extend([ + json!({ + "name": "version_id", + "type": "string", + }), + json!({ + "name": "project_id", + "type": "string", + "reference": format!("{projects_collection}.id"), + "async_reference": true, + "cascade_delete": false, + }), + json!({ + "name": "version_published_timestamp", + "type": "int64", + "sort": true, + }), + ]); + + json!({ + "name": name, + "fields": fields, + }) + } + fn text_match_sort_field(request_config: &RequestConfig) -> String { match request_config.bucketing { Bucketing::Buckets(count) => { @@ -675,12 +763,7 @@ impl Typesense { request_config: &RequestConfig, ) -> Result<(String, String), ApiError> { let sort = parse_search_index(index, new_filters)?; - let alias = match sort.index_name { - SearchIndexName::Projects => self.config.get_alias_name("projects"), - SearchIndexName::ProjectsFiltered => { - self.config.get_alias_name("projects_filtered") - } - }; + let alias = self.config.get_alias_name("projects"); Ok((alias, self.get_sort_fields(sort.index, request_config))) } @@ -689,7 +772,10 @@ impl Typesense { /// Handles the new-style filter string, legacy facets JSON, and the legacy /// `filters`/`version` fields, translating each from Meilisearch filter /// syntax to Typesense filter syntax. - fn build_filter(info: &SearchRequest) -> Result, ApiError> { + fn build_filter( + info: &SearchRequest, + versions_collection: &str, + ) -> Result, ApiError> { let facet_part = if let Some(facets_json) = info.facets.as_deref() { Some( facets_to_typesense(facets_json) @@ -710,29 +796,61 @@ impl Typesense { let filter_part = new_filters_part.or(legacy_part); - Ok(match (facet_part, filter_part) { + let filter = match (facet_part, filter_part) { (Some(f), Some(l)) if !f.is_empty() && !l.is_empty() => { Some(format!("({f}) && ({l})")) } (Some(f), _) if !f.is_empty() => Some(f), (_, Some(l)) if !l.is_empty() => Some(l), _ => None, - }) + }; + + filter + .map(|filter| { + rewrite_filter_for_join(&filter, versions_collection) + .wrap_request_err("failed to rewrite search filter") + }) + .transpose() } - /// Ensures the alias and its backing collection both exist, creating them - /// when necessary so reads succeed before the first full index run. - async fn ensure_collection(&self, alias: &str) -> Result<()> { - if self.client.get_alias(alias).await?.is_some() { - return Ok(()); - } - let name = self.config.get_next_collection_name(alias, false); - if !self.client.collection_exists(&name).await? { + async fn ensure_collections(&self) -> Result<()> { + let projects_alias = self.config.get_alias_name("projects"); + let projects_collection = if let Some(collection) = + self.client.get_alias(&projects_alias).await? + { + collection + } else { + let collection = + self.config.get_next_collection_name(&projects_alias, false); + if !self.client.collection_exists(&collection).await? { + self.client + .create_collection(&Self::project_collection_schema( + &collection, + )) + .await?; + } + self.client + .upsert_alias(&projects_alias, &collection) + .await?; + collection + }; + + let versions_alias = self.config.get_alias_name("versions"); + if self.client.get_alias(&versions_alias).await?.is_none() { + let collection = + self.config.get_next_collection_name(&versions_alias, false); + if !self.client.collection_exists(&collection).await? { + self.client + .create_collection(&Self::version_collection_schema( + &collection, + &projects_collection, + )) + .await?; + } self.client - .create_collection(&Self::collection_schema(&name)) + .upsert_alias(&versions_alias, &collection) .await?; } - self.client.upsert_alias(alias, &name).await?; Ok(()) } @@ -780,31 +898,83 @@ impl Typesense { Ok(()) } - async fn existing_write_collections(&self) -> Result> { + async fn import_version_document_batches( + &self, + collections: &[String], + documents: &[UploadSearchVersion], + ) -> Result<()> { + let batch_size = self.config.import_batch_size.max(1); + + for batch in documents.chunks(batch_size) { + let jsonl = version_documents_to_jsonl(batch)?; + + for collection in collections { + info!( + collection, + document_count = batch.len(), + content_length_bytes = jsonl.len(), + "sending Typesense version document import" + ); + self.client + .import_documents(collection, jsonl.clone()) + .await?; + } + } + + Ok(()) + } + + async fn existing_write_collections( + &self, + alias: &str, + ) -> Result> { let mut collections = Vec::new(); - for alias in [ - self.config.get_alias_name("projects"), - self.config.get_alias_name("projects_filtered"), - ] { - let live = self.client.get_alias(&alias).await?; - let shadow_alt = self.config.get_next_collection_name(&alias, true); - let shadow_current = - self.config.get_next_collection_name(&alias, false); - - for collection in - live.into_iter().chain([shadow_alt, shadow_current]) + let live = self.client.get_alias(alias).await?; + let shadow_alt = self.config.get_next_collection_name(alias, true); + let shadow_current = self.config.get_next_collection_name(alias, false); + + for collection in live.into_iter().chain([shadow_alt, shadow_current]) { + if !collections.contains(&collection) + && self.client.collection_exists(&collection).await? { - if !collections.contains(&collection) - && self.client.collection_exists(&collection).await? - { - collections.push(collection); - } + collections.push(collection); } } Ok(collections) } + + async fn delete_from_write_collections( + &self, + alias: &str, + filter: &str, + ) -> Result<()> { + for collection in self.existing_write_collections(alias).await? { + self.delete_documents_by_filter_if_exists(&collection, filter) + .await?; + } + Ok(()) + } + + async fn delete_legacy_filtered_collections(&self) -> Result<()> { + let alias = self.config.get_alias_name("projects_filtered"); + let live = self.client.get_alias(&alias).await?; + let shadow_alt = self.config.get_next_collection_name(&alias, true); + let shadow_current = + self.config.get_next_collection_name(&alias, false); + + self.client.delete_alias_if_exists(&alias).await?; + for collection in live + .into_iter() + .chain([shadow_alt, shadow_current]) + .unique() + { + self.client.delete_collection_if_exists(&collection).await?; + } + + Ok(()) + } } #[async_trait] @@ -819,7 +989,8 @@ impl SearchBackend for Typesense { info.new_filters.as_deref(), &info.typesense_config, )?; - let filter_by = Self::build_filter(info)?; + let versions_alias = self.config.get_alias_name("versions"); + let filter_by = Self::build_filter(info, &versions_alias)?; let q = if parsed.query.is_empty() { "*" @@ -857,8 +1028,6 @@ impl SearchBackend for Typesense { ("sort_by", sort_by.to_string()), ("page", parsed.page.to_string()), ("per_page", parsed.hits_per_page.to_string()), - ("group_by", "project_id".to_string()), - ("group_limit", "1".to_string()), ( "max_candidates", info.typesense_config.max_candidates.to_string(), @@ -874,6 +1043,14 @@ impl SearchBackend for Typesense { } if let Some(filter) = &filter_by { params.push(("filter_by", filter.clone())); + if filter.contains(&format!("${versions_alias}(")) { + params.push(( + "include_fields", + format!( + "${versions_alias}(version_id, sort_by: version_published_timestamp:desc, limit:1, strategy: nest_array) as matching_versions" + ), + )); + } } let resp = self @@ -916,16 +1093,33 @@ impl SearchBackend for Typesense { let total_hits = body["found"].as_u64().unwrap_or(0) as usize; - let hits = body["grouped_hits"] + let hits = body["hits"] .as_array() .cloned() .unwrap_or_default() .into_iter() - .filter_map(|group| { - let hit = group["hits"].as_array()?.first()?.clone(); + .filter_map(|hit| { let mut doc = hit.get("document")?.clone(); if let Some(obj) = doc.as_object_mut() { obj.remove("id"); + let matching_version_id = + obj.remove("matching_versions").and_then(|versions| { + versions + .as_array() + .and_then(|versions| versions.first()) + .or_else(|| { + versions.as_object().map(|_| &versions) + }) + .and_then(|version| version.get("version_id")) + .and_then(Value::as_str) + .map(ToString::to_string) + }); + if let Some(version_id) = matching_version_id { + obj.insert( + "version_id".to_string(), + Value::String(version_id), + ); + } } let metadata = info.show_metadata.then(|| { @@ -967,54 +1161,55 @@ impl SearchBackend for Typesense { info!("starting project indexing"); let projects_alias = self.config.get_alias_name("projects"); - let filtered_alias = self.config.get_alias_name("projects_filtered"); + let versions_alias = self.config.get_alias_name("versions"); - // Guarantee current aliases exist so reads keep working during re-index. - self.ensure_collection(&projects_alias).await?; - self.ensure_collection(&filtered_alias).await?; + self.ensure_collections().await?; - // Toggle the shadow collection name between __current and __alt. let projects_current = self.client.get_alias(&projects_alias).await?; - let filtered_current = self.client.get_alias(&filtered_alias).await?; + let versions_current = self.client.get_alias(&versions_alias).await?; let projects_use_alt = !projects_current .as_deref() .is_some_and(|n| n.ends_with("__alt")); - let filtered_use_alt = !filtered_current + let versions_use_alt = !versions_current .as_deref() .is_some_and(|n| n.ends_with("__alt")); let projects_next = self .config .get_next_collection_name(&projects_alias, projects_use_alt); - let filtered_next = self + let versions_next = self .config - .get_next_collection_name(&filtered_alias, filtered_use_alt); + .get_next_collection_name(&versions_alias, versions_use_alt); - info!("shadow collections `{projects_next}` and `{filtered_next}`"); + info!("shadow collections `{projects_next}` and `{versions_next}`"); self.client - .delete_collection_if_exists(&projects_next) + .delete_collection_if_exists(&versions_next) .await?; self.client - .delete_collection_if_exists(&filtered_next) + .delete_collection_if_exists(&projects_next) .await?; self.client - .create_collection(&Self::collection_schema(&projects_next)) + .create_collection(&Self::project_collection_schema(&projects_next)) .await?; self.client - .create_collection(&Self::collection_schema(&filtered_next)) + .create_collection(&Self::version_collection_schema( + &versions_next, + &projects_next, + )) .await?; let mut cursor = 0_i64; let mut chunk_idx = 0_usize; - let mut total = 0_usize; + let mut total_projects = 0_usize; + let mut total_versions = 0_usize; loop { info!("fetching index chunk {chunk_idx}"); chunk_idx += 1; - let (uploads, next_cursor) = index_local( + let (documents, next_cursor) = index_local( &ro_pool, &redis, cursor, @@ -1023,19 +1218,25 @@ impl SearchBackend for Typesense { .await .wrap_err("failed to fetch projects from local DB")?; - if uploads.is_empty() { + if documents.projects.is_empty() { info!( - "no more documents; indexed {total} in {chunk_idx} chunks" + "no more documents; indexed {total_projects} projects and {total_versions} versions in {chunk_idx} chunks" ); break; } - total += uploads.len(); + total_projects += documents.projects.len(); + total_versions += documents.versions.len(); cursor = next_cursor; self.import_document_batches( - &[projects_next.clone(), filtered_next.clone()], - &uploads, + std::slice::from_ref(&projects_next), + &documents.projects, + ) + .await?; + self.import_version_document_batches( + std::slice::from_ref(&versions_next), + &documents.versions, ) .await?; } @@ -1045,17 +1246,19 @@ impl SearchBackend for Typesense { .upsert_alias(&projects_alias, &projects_next) .await?; self.client - .upsert_alias(&filtered_alias, &filtered_next) + .upsert_alias(&versions_alias, &versions_next) .await?; info!("cleaning up old collections"); - if let Some(old) = projects_current { + if let Some(old) = versions_current { self.client.delete_collection_if_exists(&old).await?; } - if let Some(old) = filtered_current { + if let Some(old) = projects_current { self.client.delete_collection_if_exists(&old).await?; } + self.delete_legacy_filtered_collections().await?; + info!("indexing complete"); Ok(()) } @@ -1068,7 +1271,8 @@ impl SearchBackend for Typesense { return Ok(()); } - let collections = self.existing_write_collections().await?; + let alias = self.config.get_alias_name("projects"); + let collections = self.existing_write_collections(&alias).await?; debug!( ?collections, num_documents = documents.len(), @@ -1081,6 +1285,28 @@ impl SearchBackend for Typesense { Ok(()) } + async fn index_version_documents( + &self, + documents: &[UploadSearchVersion], + ) -> eyre::Result<()> { + if documents.is_empty() { + return Ok(()); + } + + let alias = self.config.get_alias_name("versions"); + let collections = self.existing_write_collections(&alias).await?; + debug!( + ?collections, + num_documents = documents.len(), + "Inserting version documents into collections", + ); + self.import_version_document_batches(&collections, documents) + .await?; + + debug!("Done importing version documents"); + Ok(()) + } + async fn remove_project_documents( &self, ids: &[ProjectId], @@ -1096,139 +1322,40 @@ impl SearchBackend for Typesense { .join(", "); let filter = format!("project_id:[{id_list}]"); - for alias in [ - self.config.get_alias_name("projects"), - self.config.get_alias_name("projects_filtered"), - ] { - debug!("Performing removal on alias {alias:?}"); - - let live = self.client.get_alias(&alias).await?; - debug!("Got live alias {live:?}"); - - let shadow_alt = self.config.get_next_collection_name(&alias, true); - debug!("Got shadow alt {shadow_alt:?}"); - - let shadow_current = - self.config.get_next_collection_name(&alias, false); - debug!("Got shadow current {shadow_current:?}"); - - let delete_live = async { - if let Some(collection) = live.as_deref() { - debug!("Working on collection {collection:?}"); - debug!( - filter_len = filter.len(), - "Collection exists, deleting by filter" - ); - self.delete_documents_by_filter_if_exists( - collection, &filter, - ) - .await?; - } - - Ok::<(), eyre::Report>(()) - }; - let delete_shadow_alt = async { - if live.as_deref() != Some(shadow_alt.as_str()) { - debug!("Working on collection {shadow_alt:?}"); - self.delete_documents_by_filter_if_exists( - &shadow_alt, - &filter, - ) - .await?; - } - - Ok::<(), eyre::Report>(()) - }; - let delete_shadow_current = async { - if live.as_deref() != Some(shadow_current.as_str()) { - debug!("Working on collection {shadow_current:?}"); - self.delete_documents_by_filter_if_exists( - &shadow_current, - &filter, - ) - .await?; - } - - Ok::<(), eyre::Report>(()) - }; - let (live_result, shadow_alt_result, shadow_current_result) = tokio::join!( - delete_live, - delete_shadow_alt, - delete_shadow_current - ); - live_result?; - shadow_alt_result?; - shadow_current_result?; - } + let projects_alias = self.config.get_alias_name("projects"); + self.delete_from_write_collections(&projects_alias, &filter) + .await?; debug!("Done"); Ok(()) } - async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()> { + async fn remove_project_version_documents( + &self, + ids: &[ProjectId], + ) -> eyre::Result<()> { if ids.is_empty() { return Ok(()); } - let id_list = ids - .iter() - .map(|id| to_base62(id.0)) - .collect::>() - .join(", "); - let filter = format!("id:[{id_list}]"); - - for alias in [ - self.config.get_alias_name("projects"), - self.config.get_alias_name("projects_filtered"), - ] { - // Delete from both the live collection and any shadow collections. - let live = self.client.get_alias(&alias).await?; - let shadow_alt = self.config.get_next_collection_name(&alias, true); - let shadow_current = - self.config.get_next_collection_name(&alias, false); - - let delete_live = async { - if let Some(collection) = live.as_deref() { - self.delete_documents_by_filter_if_exists( - collection, &filter, - ) - .await?; - } - - Ok::<(), eyre::Report>(()) - }; - let delete_shadow_alt = async { - if live.as_deref() != Some(shadow_alt.as_str()) { - self.delete_documents_by_filter_if_exists( - &shadow_alt, - &filter, - ) - .await?; - } - - Ok::<(), eyre::Report>(()) - }; - let delete_shadow_current = async { - if live.as_deref() != Some(shadow_current.as_str()) { - self.delete_documents_by_filter_if_exists( - &shadow_current, - &filter, - ) - .await?; - } + let id_list = ids.iter().map(ToString::to_string).join(", "); + let filter = format!("project_id:[{id_list}]"); + let alias = self.config.get_alias_name("versions"); + self.delete_from_write_collections(&alias, &filter).await + } - Ok::<(), eyre::Report>(()) - }; - let (live_result, shadow_alt_result, shadow_current_result) = tokio::join!( - delete_live, - delete_shadow_alt, - delete_shadow_current - ); - live_result?; - shadow_alt_result?; - shadow_current_result?; + async fn remove_version_documents( + &self, + ids: &[VersionId], + ) -> eyre::Result<()> { + if ids.is_empty() { + return Ok(()); } - Ok(()) + + let id_list = ids.iter().map(ToString::to_string).join(", "); + let filter = format!("id:[{id_list}]"); + let alias = self.config.get_alias_name("versions"); + self.delete_from_write_collections(&alias, &filter).await } async fn tasks(&self) -> eyre::Result { @@ -1246,7 +1373,7 @@ impl SearchBackend for Typesense { /// Serialises a batch of [`UploadSearchProject`]s to a JSONL string suitable /// for the Typesense bulk-import endpoint. Each document gets an `id` field -/// equal to `version_id` so Typesense can use it as the primary key. +/// equal to `project_id` so Typesense can use it as the primary key. fn documents_to_jsonl(uploads: &[UploadSearchProject]) -> Result { let mut out = String::new(); for upload in uploads { @@ -1254,7 +1381,7 @@ fn documents_to_jsonl(uploads: &[UploadSearchProject]) -> Result { .wrap_err("failed to serialise UploadSearchProject")?; if let Some(obj) = doc.as_object_mut() { let id = obj - .get("version_id") + .get("project_id") .and_then(Value::as_str) .unwrap_or_default() .to_string(); @@ -1278,6 +1405,271 @@ fn documents_to_jsonl(uploads: &[UploadSearchProject]) -> Result { Ok(out) } +fn version_documents_to_jsonl( + uploads: &[UploadSearchVersion], +) -> Result { + let mut out = String::new(); + for upload in uploads { + let mut document = serde_json::to_value(upload) + .wrap_err("failed to serialise UploadSearchVersion")?; + if let Some(object) = document.as_object_mut() { + object.insert( + "id".to_string(), + Value::String(upload.version_id.clone()), + ); + } + out.push_str(&serde_json::to_string(&document)?); + out.push('\n'); + } + Ok(out) +} + +#[derive(Clone, Default)] +struct JoinedFilterClause { + project: Vec, + version: Vec, +} + +fn rewrite_filter_for_join( + filter: &str, + versions_collection: &str, +) -> Result { + const MAX_CLAUSES: usize = 256; + + fn parse(expression: &str) -> Result> { + let expression = trim_outer_parentheses(expression.trim()); + + let or_parts = split_top_level(expression, "||"); + if or_parts.len() > 1 { + let mut clauses = Vec::new(); + for part in or_parts { + clauses.extend(parse(part)?); + if clauses.len() > MAX_CLAUSES { + return Err(eyre!( + "search filter has too many boolean clauses" + )); + } + } + return Ok(clauses); + } + + let and_parts = split_top_level(expression, "&&"); + if and_parts.len() > 1 { + let mut clauses = vec![JoinedFilterClause::default()]; + for part in and_parts { + let right = parse(part)?; + if clauses.len().saturating_mul(right.len()) > MAX_CLAUSES { + return Err(eyre!( + "search filter has too many boolean clauses" + )); + } + clauses = clauses + .into_iter() + .cartesian_product(right) + .map(|(mut left, right)| { + left.project.extend(right.project); + left.version.extend(right.version); + left + }) + .collect(); + } + return Ok(clauses); + } + + let field = filter_field(expression).ok_or_else(|| { + eyre!("could not determine filter field in `{expression}`") + })?; + let mut clause = JoinedFilterClause::default(); + if field == "categories" { + let project_expression = + expression.replacen("categories", "project_categories", 1); + if is_negative_filter(expression) { + clause.project.push(project_expression); + clause.version.push(expression.to_string()); + Ok(vec![clause]) + } else { + Ok(vec![ + JoinedFilterClause { + project: vec![project_expression], + version: Vec::new(), + }, + JoinedFilterClause { + project: Vec::new(), + version: vec![expression.to_string()], + }, + ]) + } + } else { + if is_version_filter_field(field) { + clause.version.push(expression.to_string()); + } else { + clause.project.push(expression.to_string()); + } + Ok(vec![clause]) + } + } + + let clauses = parse(filter)?; + Ok(clauses + .into_iter() + .map(|clause| { + let mut parts = clause.project; + if !clause.version.is_empty() { + parts.push(format!( + "${versions_collection}({})", + clause.version.join(" && ") + )); + } + if parts.len() == 1 { + parts.pop().unwrap_or_default() + } else { + format!("({})", parts.join(" && ")) + } + }) + .join(" || ")) +} + +fn is_version_filter_field(field: &str) -> bool { + matches!( + field, + "categories" + | "project_types" + | "environment" + | "game_versions" + | "client_side" + | "server_side" + ) +} + +fn is_negative_filter(expression: &str) -> bool { + expression + .split_once(':') + .is_some_and(|(_, value)| value.trim_start().starts_with("!=")) +} + +fn filter_field(expression: &str) -> Option<&str> { + let operator = expression.find(':')?; + let field = expression[..operator].trim(); + (!field.is_empty() + && field.chars().all(|character| { + character.is_ascii_alphanumeric() || "_.".contains(character) + })) + .then_some(field) +} + +fn trim_outer_parentheses(mut expression: &str) -> &str { + while expression.starts_with('(') + && expression.ends_with(')') + && matching_outer_parentheses(expression) + { + expression = expression[1..expression.len() - 1].trim(); + } + expression +} + +fn matching_outer_parentheses(expression: &str) -> bool { + let mut depth = 0; + let mut quote = None; + let mut escaped = false; + + for (index, character) in expression.char_indices() { + if escaped { + escaped = false; + continue; + } + if character == '\\' { + escaped = true; + continue; + } + if let Some(active_quote) = quote { + if character == active_quote { + quote = None; + } + continue; + } + if matches!(character, '\'' | '"' | '`') { + quote = Some(character); + continue; + } + match character { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 && index + character.len_utf8() < expression.len() + { + return false; + } + } + _ => {} + } + } + + depth == 0 +} + +fn split_top_level<'a>(expression: &'a str, operator: &str) -> Vec<&'a str> { + let mut parts = Vec::new(); + let mut start = 0; + let mut parentheses = 0; + let mut brackets = 0; + let mut quote = None; + let mut escaped = false; + let bytes = expression.as_bytes(); + let mut index = 0; + + while index < bytes.len() { + let character = expression[index..].chars().next().unwrap_or_default(); + let width = character.len_utf8(); + if escaped { + escaped = false; + index += width; + continue; + } + if character == '\\' { + escaped = true; + index += width; + continue; + } + if let Some(active_quote) = quote { + if character == active_quote { + quote = None; + } + index += width; + continue; + } + if matches!(character, '\'' | '"' | '`') { + quote = Some(character); + index += width; + continue; + } + match character { + '(' => parentheses += 1, + ')' => parentheses -= 1, + '[' => brackets += 1, + ']' => brackets -= 1, + _ => {} + } + + if parentheses == 0 + && brackets == 0 + && expression[index..].starts_with(operator) + { + parts.push(expression[start..index].trim()); + index += operator.len(); + start = index; + continue; + } + index += width; + } + + if parts.is_empty() { + vec![expression] + } else { + parts.push(expression[start..].trim()); + parts + } +} + /// Translates a Meilisearch filter expression into Typesense `filter_by` /// syntax. /// @@ -1410,3 +1802,61 @@ fn condition_to_typesense_filter(cond: &str) -> String { } cond.to_string() } + +#[cfg(test)] +mod tests { + use super::rewrite_filter_for_join; + + #[test] + fn project_filters_do_not_join_versions() { + assert_eq!( + rewrite_filter_for_join("license:= MIT", "versions").unwrap(), + "license:= MIT" + ); + } + + #[test] + fn correlated_version_filters_share_one_join() { + assert_eq!( + rewrite_filter_for_join( + "categories:= fabric && game_versions:= 1.21", + "versions", + ) + .unwrap(), + "(project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" + ); + } + + #[test] + fn project_and_version_filters_are_partitioned() { + assert_eq!( + rewrite_filter_for_join( + "license:= MIT && categories:= fabric", + "versions", + ) + .unwrap(), + "(license:= MIT && project_categories:= fabric) || (license:= MIT && $versions(categories:= fabric))" + ); + } + + #[test] + fn mixed_boolean_filters_preserve_version_correlation() { + assert_eq!( + rewrite_filter_for_join( + "(license:= MIT || categories:= fabric) && game_versions:= 1.21", + "versions", + ) + .unwrap(), + "(license:= MIT && $versions(game_versions:= 1.21)) || (project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" + ); + } + + #[test] + fn negative_categories_require_project_and_version_exclusion() { + assert_eq!( + rewrite_filter_for_join("categories:!= fabric", "versions") + .unwrap(), + "(project_categories:!= fabric && $versions(categories:!= fabric))" + ); + } +} diff --git a/apps/labrinth/src/search/incremental.rs b/apps/labrinth/src/search/incremental.rs index 96e8b7e096..a418e7638c 100644 --- a/apps/labrinth/src/search/incremental.rs +++ b/apps/labrinth/src/search/incremental.rs @@ -36,7 +36,11 @@ impl IncrementalSearchQueue { } } - pub async fn push( + pub async fn push(&self, project_id: ProjectId) { + self.operations.lock().await.push_project_change(project_id); + } + + pub async fn push_versions( &self, project_id: ProjectId, version_ids: impl IntoIterator, @@ -44,7 +48,7 @@ impl IncrementalSearchQueue { self.operations .lock() .await - .push_project_change(project_id, version_ids); + .push_version_change(project_id, version_ids); } pub async fn push_project_removal(&self, project_id: ProjectId) { @@ -123,22 +127,27 @@ impl PendingSearchIndexOperations { && self.removed_project_ids.is_empty() } - fn push_project_change( + fn push_project_change(&mut self, project_id: ProjectId) { + if !self.removed_project_ids.contains(&project_id) { + self.changed_project_ids.insert(project_id); + } + } + + fn push_version_change( &mut self, project_id: ProjectId, version_ids: impl IntoIterator, ) { - if !self.removed_project_ids.contains(&project_id) { - let version_ids = version_ids.into_iter().collect::>(); - if version_ids.is_empty() { - self.changed_project_versions.remove(&project_id); - self.changed_project_ids.insert(project_id); - } else if !self.changed_project_ids.contains(&project_id) { - self.changed_project_versions - .entry(project_id) - .or_default() - .extend(version_ids); - } + if self.removed_project_ids.contains(&project_id) { + return; + } + + let version_ids = version_ids.into_iter().collect::>(); + if !version_ids.is_empty() { + self.changed_project_versions + .entry(project_id) + .or_default() + .extend(version_ids); } } @@ -151,16 +160,12 @@ impl PendingSearchIndexOperations { fn push_event(&mut self, event: SearchProjectIndexQueueEventData) { match event { SearchProjectIndexQueueEventData::Change { project_id } => { - self.push_project_change(project_id, []) + self.push_project_change(project_id) } SearchProjectIndexQueueEventData::VersionChange { project_id, version_ids, - } => { - if !version_ids.is_empty() { - self.push_project_change(project_id, version_ids) - } - } + } => self.push_version_change(project_id, version_ids), SearchProjectIndexQueueEventData::Removal { project_id } => { self.push_project_removal(project_id) } @@ -188,7 +193,6 @@ impl PendingSearchIndexOperations { } }, )); - events } } diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 2cfd315f24..94f5274169 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -191,7 +191,9 @@ async fn consume_batch( .retain(|project_id| !project_ids_to_remove.contains(project_id)); project_ids_with_version_changes .retain(|project_id| !project_ids_to_remove.contains(project_id)); - + project_ids_to_change.retain(|project_id| { + !project_ids_with_version_changes.contains(project_id) + }); let project_ids_to_change = project_ids_to_change.into_iter().collect::>(); let project_ids_with_version_changes = project_ids_with_version_changes @@ -204,7 +206,7 @@ async fn consume_batch( info!( kafka.message_count = messages_to_commit.len(), - "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with version changes, {} versions to change, and {} projects to remove", + "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with {} version changes, and {} projects to remove", start.elapsed(), project_ids_to_change.len(), project_ids_with_version_changes.len(), @@ -219,6 +221,10 @@ async fn consume_batch( project_count = project_ids_to_remove.len(), "Removing project documents" ); + search_backend + .remove_project_version_documents(&project_ids_to_remove) + .await + .wrap_err("failed to remove project version documents")?; search_backend .remove_project_documents(&project_ids_to_remove) .await @@ -232,12 +238,8 @@ async fn consume_batch( if !version_ids_to_change.is_empty() { let operation_start = Instant::now(); - info!( - version_count = version_ids_to_change.len(), - "Removing changed version documents", - ); search_backend - .remove_documents(&version_ids_to_change) + .remove_version_documents(&version_ids_to_change) .await .wrap_err("failed to remove changed version documents")?; info!( @@ -249,12 +251,7 @@ async fn consume_batch( if !project_ids_with_version_changes.is_empty() { let operation_start = Instant::now(); - info!( - project_count = project_ids_with_version_changes.len(), - version_count = version_ids_to_change.len(), - "Indexing changed project versions" - ); - index_changed_project_versions( + reindex_changed_project_versions( ro_pool, redis_pool, search_backend, @@ -262,11 +259,11 @@ async fn consume_batch( &version_ids_to_change, ) .await - .wrap_err("failed to index changed project version batch")?; + .wrap_err("failed to reindex changed project versions")?; info!( project_count = project_ids_with_version_changes.len(), version_count = version_ids_to_change.len(), - "Indexed changed project versions in {:.2?}", + "Reindexed changed project versions in {:.2?}", operation_start.elapsed() ); } @@ -275,19 +272,19 @@ async fn consume_batch( let operation_start = Instant::now(); info!( project_count = project_ids_to_change.len(), - "Indexing changed projects" + "Reindexing changed projects" ); - index_changed_projects( + reindex_projects( ro_pool, redis_pool, search_backend, &project_ids_to_change, ) .await - .wrap_err("failed to index changed project batch")?; + .wrap_err("failed to reindex changed project batch")?; info!( project_count = project_ids_to_change.len(), - "Indexed changed projects in {:.2?}", + "Reindexed changed projects in {:.2?}", operation_start.elapsed() ); } @@ -356,7 +353,7 @@ async fn index_changed_projects( Ok(()) } -async fn index_changed_project_versions( +async fn reindex_changed_project_versions( ro_pool: &PgPool, redis_pool: &RedisPool, search_backend: &dyn SearchBackend, @@ -383,9 +380,11 @@ async fn index_changed_project_versions( ) })?; - info!("Fetched all project version documents, indexing into backend"); - - search_backend.index_documents(&documents).await?; + search_backend.remove_project_documents(project_ids).await?; + search_backend.index_documents(&documents.projects).await?; + search_backend + .index_version_documents(&documents.versions) + .await?; Ok(()) } diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 503e34e002..792aa78eea 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -24,7 +24,10 @@ use crate::models::ids::{ProjectId, VersionId}; use crate::models::projects::{DependencyType, from_duplicate_version_fields}; use crate::models::v2::projects::LegacyProject; use crate::routes::v2_reroute; -use crate::search::{SearchProjectDependency, UploadSearchProject}; +use crate::search::{ + SearchDocumentBatch, SearchProjectDependency, UploadSearchProject, + UploadSearchVersion, +}; use crate::util::error::Context; struct PartialProject { @@ -68,7 +71,7 @@ pub async fn index_local( redis: &RedisPool, cursor: i64, limit: i64, -) -> eyre::Result<(Vec, i64)> { +) -> eyre::Result<(SearchDocumentBatch, i64)> { info!("Indexing local projects!"); let searchable_statuses = searchable_statuses(); @@ -111,12 +114,11 @@ pub async fn index_local( let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); let Some(largest) = project_ids.iter().max() else { - return Ok((vec![], i64::MAX)); + return Ok((SearchDocumentBatch::default(), i64::MAX)); }; - let uploads = - build_search_documents(pool, redis, db_projects, None).await?; - Ok((uploads, *largest)) + let documents = build_search_documents(pool, redis, db_projects).await?; + Ok((documents, *largest)) } pub async fn index_project_documents( @@ -164,7 +166,9 @@ pub async fn index_project_documents( info!("Fetched partial projects"); - build_search_documents(pool, redis, db_projects, None).await + Ok(build_search_documents(pool, redis, db_projects) + .await? + .projects) } pub async fn index_project_version_documents( @@ -172,16 +176,33 @@ pub async fn index_project_version_documents( redis: &RedisPool, project_ids: &[ProjectId], version_ids: &[VersionId], -) -> eyre::Result> { +) -> eyre::Result { + let projects = + index_project_document_batch(pool, redis, project_ids).await?; + let version_ids = version_ids + .iter() + .map(ToString::to_string) + .collect::>(); + Ok(SearchDocumentBatch { + projects: projects.projects, + versions: projects + .versions + .into_iter() + .filter(|version| version_ids.contains(&version.version_id)) + .collect(), + }) +} + +async fn index_project_document_batch( + pool: &PgPool, + redis: &RedisPool, + project_ids: &[ProjectId], +) -> eyre::Result { let searchable_statuses = searchable_statuses(); let project_ids = project_ids .iter() .map(|project_id| DBProjectId::from(*project_id).0) .collect::>(); - let version_ids = version_ids - .iter() - .map(|version_id| DBVersionId::from(*version_id)) - .collect::>(); let db_projects = sqlx::query!( r#" @@ -215,15 +236,14 @@ pub async fn index_project_version_documents( .await .wrap_err("failed to fetch project")?; - build_search_documents(pool, redis, db_projects, Some(&version_ids)).await + build_search_documents(pool, redis, db_projects).await } async fn build_search_documents( pool: &PgPool, redis: &RedisPool, db_projects: Vec, - version_ids_to_index: Option<&HashSet>, -) -> eyre::Result> { +) -> eyre::Result { let searchable_statuses = searchable_statuses(); let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); let project_components = db_projects @@ -391,7 +411,7 @@ async fn build_search_documents( .await?; info!("Getting all loader fields!"); - let loader_fields: Vec = sqlx::query!( + let loader_field_definitions: Vec = sqlx::query!( " SELECT DISTINCT id, field, field_type, enum_type, min_val, max_val, optional FROM loader_fields lf @@ -409,7 +429,8 @@ async fn build_search_documents( }) .try_collect() .await?; - let loader_fields: Vec<&QueryLoaderField> = loader_fields.iter().collect(); + let loader_field_definitions: Vec<&QueryLoaderField> = + loader_field_definitions.iter().collect(); info!("Getting all loader field enum values!"); @@ -434,7 +455,8 @@ async fn build_search_documents( .await?; info!("Indexing loaders, project types!"); - let mut uploads = Vec::new(); + let mut project_uploads = Vec::new(); + let mut version_uploads = Vec::new(); let total_len = db_projects.len(); let mut count = 0; @@ -533,21 +555,34 @@ async fn build_search_documents( .collect::>(); if let Some(versions) = versions.remove(&project.id) { - // Aggregated project loader fields + let Some(latest_version) = versions.iter().max_by(|a, b| { + a.date_published + .cmp(&b.date_published) + .then_with(|| a.id.0.cmp(&b.id.0)) + }) else { + continue; + }; + let project_version_fields = versions .iter() .flat_map(|x| x.version_fields.clone()) .collect::>(); let aggregated_version_fields = VersionField::from_query_json( project_version_fields, - &loader_fields, + &loader_field_definitions, &loader_field_enum_values, true, ); - let project_loader_fields = + let unvectorized_loader_fields = aggregated_version_fields + .iter() + .map(|field| { + (field.field_name.clone(), field.value.serialize_internal()) + }) + .collect(); + let mut loader_fields = from_duplicate_version_fields(aggregated_version_fields); + let project_loader_fields = loader_fields.clone(); - // aggregated project loaders let mut project_loaders = versions .iter() .flat_map(|x| x.loaders.clone()) @@ -555,162 +590,184 @@ async fn build_search_documents( project_loaders.sort(); project_loaders.dedup(); - // all valid project types across every version of the project, so that - // filters can exclude projects that have *any* version of a given - // project type (unlike the version-specific `project_types` field). - let mut all_project_types = versions + let mut project_types = versions .iter() .flat_map(|x| x.project_types.clone()) .collect::>(); - all_project_types.sort(); - all_project_types.dedup(); + project_types.sort(); + project_types.dedup(); exp::compat::correct_project_types( &project.components, - &mut all_project_types, + &mut project_types, ); - for version in versions { - if let Some(version_ids_to_index) = version_ids_to_index - && !version_ids_to_index.contains(&version.id) - { - continue; - } - + let project_id = ProjectId::from(project.id).to_string(); + version_uploads.extend(versions.iter().map(|version| { let version_fields = VersionField::from_query_json( - version.version_fields, - &loader_fields, + version.version_fields.clone(), + &loader_field_definitions, &loader_field_enum_values, false, ); let unvectorized_loader_fields = version_fields .iter() - .map(|vf| { - (vf.field_name.clone(), vf.value.serialize_internal()) + .map(|field| { + ( + field.field_name.clone(), + field.value.serialize_internal(), + ) }) .collect(); - let mut loader_fields = - from_duplicate_version_fields(version_fields); - let mut project_types = version.project_types; - + let mut fields = from_duplicate_version_fields(version_fields); + let mut version_project_types = version.project_types.clone(); exp::compat::correct_project_types( &project.components, - &mut project_types, + &mut version_project_types, ); - let mut version_loaders = version.loaders; - - // Uses version loaders, not project loaders. - let mut categories = categories.clone(); - categories.append(&mut version_loaders.clone()); - - let display_categories = display_categories.clone(); - categories.append(&mut version_loaders); - - // SPECIAL BEHAVIOUR - // Todo: revisit. - // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category. - // These were previously considered the loader, and in v2, the loader is a category for searching. - // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories. - // The loaders are kept in loader_fields as well, so that no information is lost on retrieval. - let mrpack_loaders = loader_fields + let mut version_categories = version.loaders.clone(); + let mrpack_loaders = fields .get("mrpack_loaders") - .cloned() - .map(|x| { - x.into_iter() - .filter_map(|x| x.as_str().map(String::from)) - .collect::>() - }) - .unwrap_or_default(); - categories.extend(mrpack_loaders); - if loader_fields.contains_key("mrpack_loaders") { - categories.retain(|x| *x != "mrpack"); + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(String::from)) + .collect::>(); + version_categories.extend(mrpack_loaders); + if fields.contains_key("mrpack_loaders") { + version_categories.retain(|category| category != "mrpack"); } + version_categories.sort(); + version_categories.dedup(); - // SPECIAL BEHAVIOUR: - // For consistency with v2 searching, we manually input the - // client_side and server_side fields from the loader fields into - // separate loader fields. - // 'client_side' and 'server_side' remain supported by meilisearch even though they are no longer v3 fields. let (_, v2_og_project_type) = - LegacyProject::get_project_type(&project_types); + LegacyProject::get_project_type(&version_project_types); let (client_side, server_side) = v2_reroute::convert_v3_side_types_to_v2_side_types( &unvectorized_loader_fields, Some(&v2_og_project_type), ); - if let Ok(client_side) = serde_json::to_value(client_side) { - loader_fields - .insert("client_side".to_string(), vec![client_side]); + fields.insert("client_side".to_string(), vec![client_side]); } if let Ok(server_side) = serde_json::to_value(server_side) { - loader_fields - .insert("server_side".to_string(), vec![server_side]); + fields.insert("server_side".to_string(), vec![server_side]); } - - let components = project - .components - .clone() - .into_query( - ProjectId::from(project.id), - &project_query_context, + fields.retain(|field, _| { + matches!( + field.as_str(), + "environment" + | "game_versions" + | "client_side" + | "server_side" ) - .wrap_err("failed to populate query components")?; - - let usp = UploadSearchProject { - version_id: crate::models::ids::VersionId::from(version.id) - .to_string(), - project_id: crate::models::ids::ProjectId::from(project.id) - .to_string(), - name: project.name.clone(), - indexed_name: normalize_for_search(&project.name), - summary: project.summary.clone(), - categories: categories.clone(), - display_categories: display_categories.clone(), - follows: project.follows, - downloads: project.downloads, - log_downloads: (project.downloads.max(1) as f64).ln(), - icon_url: project.icon_url.clone(), - author: username.clone(), - author_id: ariadne::ids::UserId::from(user_id).to_string(), - organization: org_name.clone(), - organization_id: org_id.map(|e| { - crate::models::ids::OrganizationId::from(e).to_string() - }), - indexed_author: normalize_for_search(&username), - date_created: project.approved, - created_timestamp: project.approved.timestamp(), - date_modified: project.updated, - modified_timestamp: project.updated.timestamp(), + }); + + UploadSearchVersion { + version_id: VersionId::from(version.id).to_string(), + project_id: project_id.clone(), + categories: version_categories, + project_types: version_project_types, version_published_timestamp: version .date_published .timestamp(), - license: license.clone(), - slug: project.slug.clone(), - // TODO - project_types, - all_project_types: all_project_types.clone(), - gallery: gallery.clone(), - featured_gallery: featured_gallery.clone(), - open_source, - color: project.color.map(|x| x as u32), - dependency_project_ids: dependency_project_ids.clone(), - compatible_dependency_project_ids: - compatible_dependency_project_ids.clone(), - dependencies: dependencies.clone(), - loader_fields, - project_loader_fields: project_loader_fields.clone(), - // 'loaders' is aggregate of all versions' loaders - loaders: project_loaders.clone(), - components, - }; + loader_fields: fields, + } + })); - uploads.push(usp); + let project_categories = categories.clone(); + let mut categories = categories; + categories.extend(project_loaders.iter().cloned()); + + let mrpack_loaders = loader_fields + .get("mrpack_loaders") + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(String::from)) + .collect::>(); + categories.extend(mrpack_loaders); + if loader_fields.contains_key("mrpack_loaders") { + categories.retain(|category| category != "mrpack"); + } + categories.sort(); + categories.dedup(); + + let (_, v2_og_project_type) = + LegacyProject::get_project_type(&project_types); + let (client_side, server_side) = + v2_reroute::convert_v3_side_types_to_v2_side_types( + &unvectorized_loader_fields, + Some(&v2_og_project_type), + ); + + if let Ok(client_side) = serde_json::to_value(client_side) { + loader_fields + .insert("client_side".to_string(), vec![client_side]); } + if let Ok(server_side) = serde_json::to_value(server_side) { + loader_fields + .insert("server_side".to_string(), vec![server_side]); + } + + let components = project + .components + .clone() + .into_query(ProjectId::from(project.id), &project_query_context) + .wrap_err("failed to populate query components")?; + let indexed_name = normalize_for_search(&project.name); + + project_uploads.push(UploadSearchProject { + version_id: crate::models::ids::VersionId::from( + latest_version.id, + ) + .to_string(), + project_id, + name: project.name, + indexed_name, + summary: project.summary, + categories, + project_categories, + display_categories, + follows: project.follows, + downloads: project.downloads, + log_downloads: (project.downloads.max(1) as f64).ln(), + icon_url: project.icon_url, + author: username.clone(), + author_id: ariadne::ids::UserId::from(user_id).to_string(), + organization: org_name, + organization_id: org_id.map(|id| { + crate::models::ids::OrganizationId::from(id).to_string() + }), + indexed_author: normalize_for_search(&username), + date_created: project.approved, + created_timestamp: project.approved.timestamp(), + date_modified: project.updated, + modified_timestamp: project.updated.timestamp(), + version_published_timestamp: latest_version + .date_published + .timestamp(), + license, + slug: project.slug, + project_types: project_types.clone(), + all_project_types: project_types, + gallery, + featured_gallery, + open_source, + color: project.color.map(|x| x as u32), + dependency_project_ids, + compatible_dependency_project_ids, + dependencies, + project_loader_fields, + loader_fields, + loaders: project_loaders, + components, + }); } } - Ok(uploads) + Ok(SearchDocumentBatch { + projects: project_uploads, + versions: version_uploads, + }) } struct PartialVersion { diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 7b1db63299..0b53f387ea 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -116,12 +116,25 @@ pub trait SearchBackend: Send + Sync { documents: &[UploadSearchProject], ) -> eyre::Result<()>; + async fn index_version_documents( + &self, + documents: &[UploadSearchVersion], + ) -> eyre::Result<()>; + async fn remove_project_documents( &self, ids: &[ProjectId], ) -> eyre::Result<()>; - async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>; + async fn remove_project_version_documents( + &self, + ids: &[ProjectId], + ) -> eyre::Result<()>; + + async fn remove_version_documents( + &self, + ids: &[VersionId], + ) -> eyre::Result<()>; async fn tasks(&self) -> eyre::Result; @@ -238,6 +251,7 @@ impl FromStr for SearchBackendKind { /// serialized as `null`. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct UploadSearchProject { + /// ID of the most recently published version. pub version_id: String, pub project_id: String, // @@ -256,6 +270,7 @@ pub struct UploadSearchProject { pub indexed_name: String, pub summary: String, pub categories: Vec, + pub project_categories: Vec, pub display_categories: Vec, pub follows: i32, pub downloads: i32, @@ -274,7 +289,7 @@ pub struct UploadSearchProject { pub date_modified: DateTime, /// Unix timestamp of the last major modification pub modified_timestamp: i64, - /// Unix timestamp of the publication date of the version + /// Unix timestamp of the most recently published version. pub version_published_timestamp: i64, pub open_source: bool, #[serde(skip_serializing_if = "Option::is_none")] @@ -296,6 +311,23 @@ pub struct UploadSearchProject { pub loader_fields: HashMap>, } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct UploadSearchVersion { + pub version_id: String, + pub project_id: String, + pub categories: Vec, + pub project_types: Vec, + pub version_published_timestamp: i64, + #[serde(flatten)] + pub loader_fields: HashMap>, +} + +#[derive(Debug, Default)] +pub struct SearchDocumentBatch { + pub projects: Vec, + pub versions: Vec, +} + /// Nullable fields in Typesense-bound documents should use /// `skip_serializing_if = "Option::is_none"` so they are omitted instead of /// serialized as `null`. @@ -320,6 +352,7 @@ pub struct SearchResults { #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] pub struct ResultSearchProject { + /// ID of the most recently published version. pub version_id: String, pub project_id: String, pub project_types: Vec, diff --git a/scripts/convert-typesense-project-docs.py b/scripts/convert-typesense-project-docs.py new file mode 100755 index 0000000000..5cfd594028 --- /dev/null +++ b/scripts/convert-typesense-project-docs.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Split legacy Typesense JSONL into project and version documents.""" + +import argparse +import json +import os +import shutil +import sys +import tempfile +import time +import zlib +from collections import OrderedDict + + +VERSION_FILTER_PATHS = ( + "project_types", + "environment", + "game_versions", + "client_side", + "server_side", +) + +BASE62_DIGITS = { + character: index + for index, character in enumerate( + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + ) +} + + +class ConversionError(Exception): + pass + + +def parse_args(): + parser = argparse.ArgumentParser( + description=( + "Convert a legacy Typesense JSONL export from one document per " + "version into separate project and version collections." + ) + ) + parser.add_argument("input", help="Legacy Typesense JSONL export") + parser.add_argument("projects_output", help="Destination project-document JSONL") + parser.add_argument("versions_output", help="Destination version-document JSONL") + parser.add_argument( + "--shards", + type=int, + default=512, + help="Temporary hash shards used to bound memory usage (default: 512)", + ) + parser.add_argument( + "--max-open-shards", + type=int, + default=64, + help="Maximum temporary shard files held open at once (default: 64)", + ) + parser.add_argument( + "--progress-interval", + type=float, + default=10.0, + help="Seconds between progress reports (default: 10)", + ) + parser.add_argument( + "--keep-temporary", + action="store_true", + help="Keep temporary shard files after completion or failure", + ) + return parser.parse_args() + + +def base62_value(value): + result = 0 + try: + for character in value: + result = result * 62 + BASE62_DIGITS[character] + except KeyError: + return -1 + return result + + +def get_path(document, path): + value = document + for segment in path.split("."): + if not isinstance(value, dict) or segment not in value: + return None, False + value = value[segment] + return value, True + + +def set_path(document, path, value): + segments = path.split(".") + current = document + for segment in segments[:-1]: + child = current.get(segment) + if not isinstance(child, dict): + child = {} + current[segment] = child + current = child + current[segments[-1]] = value + + +def unique_sorted(values): + return sorted(set(values)) + + +def extend_values(target, value): + if isinstance(value, list): + target.extend(value) + elif value is not None: + target.append(value) + + +def version_filter_document(document): + version_id = document.get("version_id") or document.get("id") + result = { + "id": str(version_id), + "version_id": str(version_id), + "project_id": str(document["project_id"]), + "version_published_timestamp": document.get( + "version_published_timestamp", -1 + ), + } + loaders = list(document.get("loaders") or []) + mrpack_loaders = list(document.get("mrpack_loaders") or []) + loaders.extend(mrpack_loaders) + if mrpack_loaders: + loaders = [loader for loader in loaders if loader != "mrpack"] + result["categories"] = unique_sorted(loaders) + for path in VERSION_FILTER_PATHS: + value, exists = get_path(document, path) + if exists and value is not None: + set_path(result, path, value) + return result + + +class ProjectAccumulator: + def __init__(self, document): + self.latest_document = document + self.latest_key = self.version_key(document) + self.versions = {} + self.categories = [] + self.version_categories = [] + self.loaders = [] + self.project_types = [] + self.client_side = [] + self.server_side = [] + self.add(document) + + @staticmethod + def version_key(document): + return ( + document.get("version_published_timestamp", -1), + base62_value(str(document.get("version_id", ""))), + ) + + def add(self, document): + project_id = document.get("project_id") + if project_id != self.latest_document.get("project_id"): + raise ConversionError("attempted to combine different projects") + + version_id = document.get("version_id") or document.get("id") + if not version_id: + raise ConversionError(f"project `{project_id}` has a version without an ID") + + key = self.version_key(document) + if key > self.latest_key: + self.latest_document = document + self.latest_key = key + + version_document = version_filter_document(document) + self.versions[str(version_id)] = (key, version_document) + extend_values(self.categories, document.get("categories")) + extend_values(self.version_categories, version_document.get("categories")) + extend_values(self.loaders, document.get("loaders")) + extend_values(self.project_types, document.get("project_types")) + extend_values(self.project_types, document.get("all_project_types")) + extend_values(self.client_side, document.get("client_side")) + extend_values(self.server_side, document.get("server_side")) + + def finish(self): + result = dict(self.latest_document) + project_id = str(result["project_id"]) + all_project_types = unique_sorted(self.project_types) + + result["id"] = project_id + result["version_id"] = str( + self.latest_document.get("version_id") + or self.latest_document.get("id") + ) + result["categories"] = unique_sorted(self.categories) + result["project_categories"] = unique_sorted( + set(self.categories) - set(self.version_categories) + ) + result["loaders"] = unique_sorted(self.loaders) + result["project_types"] = all_project_types + result["all_project_types"] = all_project_types + + project_loader_fields = result.get("project_loader_fields") + if not isinstance(project_loader_fields, dict): + project_loader_fields = {} + result["project_loader_fields"] = project_loader_fields + for field, value in project_loader_fields.items(): + result[field] = value + + if self.client_side: + result["client_side"] = unique_sorted(self.client_side) + if self.server_side: + result["server_side"] = unique_sorted(self.server_side) + + versions = [ + version + for _, version in sorted( + self.versions.values(), key=lambda item: item[0] + ) + ] + result.pop("versions", None) + return result, versions + + +class ShardWriter: + def __init__(self, directory, shard_count, max_open): + self.directory = directory + self.shard_count = shard_count + self.max_open = max_open + self.handles = OrderedDict() + + def path(self, shard): + return os.path.join(self.directory, f"shard-{shard:04d}.jsonl") + + def write(self, project_id, line): + shard = zlib.crc32(project_id.encode("utf-8")) % self.shard_count + handle = self.handles.pop(shard, None) + if handle is None: + if len(self.handles) >= self.max_open: + _, oldest = self.handles.popitem(last=False) + oldest.close() + handle = open(self.path(shard), "ab") + self.handles[shard] = handle + handle.write(line) + + def close(self): + for handle in self.handles.values(): + handle.close() + self.handles.clear() + + +def shard_input(args, temporary_directory): + writer = ShardWriter( + temporary_directory, + args.shards, + args.max_open_shards, + ) + input_size = os.path.getsize(args.input) + bytes_read = 0 + document_count = 0 + started_at = time.monotonic() + last_report = started_at + + try: + with open(args.input, "rb") as input_file: + for line_number, line in enumerate(input_file, start=1): + bytes_read += len(line) + if not line.strip(): + continue + try: + document = json.loads(line) + except json.JSONDecodeError as error: + raise ConversionError( + f"invalid JSON on input line {line_number}: {error}" + ) from error + project_id = document.get("project_id") + if not project_id: + raise ConversionError( + f"input line {line_number} has no `project_id`" + ) + writer.write(str(project_id), line) + document_count += 1 + + now = time.monotonic() + if now - last_report >= args.progress_interval: + percent = bytes_read / input_size * 100 if input_size else 100 + print( + f"sharding: {percent:.1f}% ({document_count:,} documents)", + flush=True, + ) + last_report = now + finally: + writer.close() + + print( + f"sharding complete: {document_count:,} version documents in " + f"{time.monotonic() - started_at:.1f}s", + flush=True, + ) + return document_count + + +def convert_shards( + args, + temporary_directory, + partial_projects_output, + partial_versions_output, +): + project_count = 0 + version_count = 0 + started_at = time.monotonic() + last_report = started_at + + with ( + open(partial_projects_output, "w", encoding="utf-8") as projects_file, + open(partial_versions_output, "w", encoding="utf-8") as versions_file, + ): + for shard in range(args.shards): + path = os.path.join(temporary_directory, f"shard-{shard:04d}.jsonl") + if not os.path.exists(path): + continue + + projects = {} + with open(path, "rb") as shard_file: + for line_number, line in enumerate(shard_file, start=1): + try: + document = json.loads(line) + except json.JSONDecodeError as error: + raise ConversionError( + f"invalid JSON in shard {shard}, line {line_number}: {error}" + ) from error + project_id = str(document["project_id"]) + if project_id in projects: + projects[project_id].add(document) + else: + projects[project_id] = ProjectAccumulator(document) + version_count += 1 + + for project_id in sorted(projects): + project, versions = projects[project_id].finish() + json.dump( + project, + projects_file, + separators=(",", ":"), + ensure_ascii=False, + ) + projects_file.write("\n") + for version in versions: + json.dump( + version, + versions_file, + separators=(",", ":"), + ensure_ascii=False, + ) + versions_file.write("\n") + project_count += 1 + + os.remove(path) + now = time.monotonic() + if now - last_report >= args.progress_interval: + print( + f"converting: shard {shard + 1}/{args.shards}, " + f"{project_count:,} projects", + flush=True, + ) + last_report = now + + print( + f"conversion complete: {version_count:,} versions into " + f"{project_count:,} projects in {time.monotonic() - started_at:.1f}s", + flush=True, + ) + return project_count, version_count + + +def main(): + args = parse_args() + if args.shards <= 0: + raise ConversionError("--shards must be greater than zero") + if args.max_open_shards <= 0: + raise ConversionError("--max-open-shards must be greater than zero") + if args.progress_interval <= 0: + raise ConversionError("--progress-interval must be greater than zero") + if not os.path.isfile(args.input): + raise ConversionError(f"input file does not exist: {args.input}") + paths = { + os.path.abspath(args.input), + os.path.abspath(args.projects_output), + os.path.abspath(args.versions_output), + } + if len(paths) != 3: + raise ConversionError("input and output paths must all differ") + + projects_directory = os.path.dirname(os.path.abspath(args.projects_output)) + versions_directory = os.path.dirname(os.path.abspath(args.versions_output)) + os.makedirs(projects_directory, exist_ok=True) + os.makedirs(versions_directory, exist_ok=True) + temporary_directory = tempfile.mkdtemp( + prefix="typesense-project-convert-", + dir=projects_directory, + ) + partial_projects_output = f"{args.projects_output}.partial" + partial_versions_output = f"{args.versions_output}.partial" + + try: + expected_versions = shard_input(args, temporary_directory) + project_count, version_count = convert_shards( + args, + temporary_directory, + partial_projects_output, + partial_versions_output, + ) + if version_count != expected_versions: + raise ConversionError( + f"sharded {expected_versions} versions but converted {version_count}" + ) + os.replace(partial_projects_output, args.projects_output) + os.replace(partial_versions_output, args.versions_output) + print( + f"wrote {project_count:,} project documents to " + f"{args.projects_output} and {version_count:,} version documents to " + f"{args.versions_output}", + flush=True, + ) + except Exception: + for partial_output in ( + partial_projects_output, + partial_versions_output, + ): + if os.path.exists(partial_output): + os.remove(partial_output) + raise + finally: + if args.keep_temporary: + print(f"temporary shards kept at {temporary_directory}", file=sys.stderr) + else: + shutil.rmtree(temporary_directory, ignore_errors=True) + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("interrupted", file=sys.stderr) + sys.exit(130) + except (ConversionError, OSError) as error: + print(error, file=sys.stderr) + sys.exit(1) From 60cf5cb147bdd9f0c0822d84a490550536b6c91b Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Wed, 15 Jul 2026 20:31:08 +0100 Subject: [PATCH 20/26] clean up SearchBackend interface --- apps/labrinth/src/routes/v3/projects.rs | 16 +- apps/labrinth/src/routes/v3/versions.rs | 15 +- .../src/search/backend/typesense/mod.rs | 171 +++++++++--------- .../src/search/incremental/consume.rs | 141 ++++++++------- apps/labrinth/src/search/mod.rs | 35 ++-- 5 files changed, 186 insertions(+), 192 deletions(-) diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index 43ced679ee..1e388aec5f 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -29,7 +29,8 @@ use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::routes::internal::delphi; use crate::search::{ - SearchBackend, SearchQuery, SearchRequest, SearchResults, SearchState, + SearchBackend, SearchIndexUpdate, SearchQuery, SearchRequest, + SearchResults, SearchState, }; use crate::util::error::Context; use crate::util::img; @@ -2849,16 +2850,13 @@ pub async fn project_delete_internal( &redis, ) .await?; + let project_id = project.inner.id.into(); search_state .backend - .remove_project_version_documents(&[project.inner.id.into()]) - .await - .wrap_internal_err( - "failed to remove project versions from search index", - )?; - search_state - .backend - .remove_project_documents(&[project.inner.id.into()]) + .apply_update(SearchIndexUpdate { + removed_projects: std::slice::from_ref(&project_id), + ..SearchIndexUpdate::default() + }) .await .wrap_internal_err("failed to remove project from search index")?; search_state diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index f912956fd1..da5d04f6e2 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -27,7 +27,7 @@ use crate::models::teams::ProjectPermissions; use crate::queue::file_scan::get_files_missing_attribution; use crate::queue::session::AuthQueue; use crate::routes::internal::delphi; -use crate::search::incremental::consume::reindex_project; +use crate::search::incremental::consume::reindex_project_versions; use crate::search::{SearchBackend, SearchState}; use crate::util::error::Context; use crate::util::img; @@ -1257,18 +1257,17 @@ pub async fn version_delete( [VersionId::from(version.inner.id)], ) .await?; - search_backend - .remove_version_documents(&[version.inner.id.into()]) - .await - .wrap_internal_err("failed to remove version search document")?; - reindex_project( + let project_id = version.inner.project_id.into(); + let version_id = version.inner.id.into(); + reindex_project_versions( &pool, &redis, search_backend.as_ref(), - version.inner.project_id.into(), + std::slice::from_ref(&project_id), + std::slice::from_ref(&version_id), ) .await - .wrap_internal_err("failed to reindex project")?; + .wrap_internal_err("failed to update search index after version removal")?; if result.is_some() { Ok(HttpResponse::NoContent().body("")) } else { diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index e8d00b9f5f..7222a4006f 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -1,6 +1,5 @@ use std::sync::LazyLock; -use ariadne::ids::base62_impl::to_base62; use async_trait::async_trait; use eyre::{Result, eyre}; use itertools::Itertools; @@ -13,7 +12,6 @@ use tracing::{debug, info}; use crate::database::PgPool; use crate::database::redis::RedisPool; use crate::env::ENV; -use crate::models::ids::{ProjectId, VersionId}; use crate::routes::ApiError; use crate::search::backend::{ SearchIndex, combined_search_filters, parse_search_index, @@ -21,8 +19,9 @@ use crate::search::backend::{ }; use crate::search::indexing::index_local; use crate::search::{ - ResultSearchProject, SearchBackend, SearchField, SearchRequest, - SearchResults, TasksCancelFilter, UploadSearchProject, UploadSearchVersion, + ResultSearchProject, SearchBackend, SearchField, SearchIndexUpdate, + SearchRequest, SearchResults, TasksCancelFilter, UploadSearchProject, + UploadSearchVersion, }; use crate::util::error::Context; @@ -959,6 +958,19 @@ impl Typesense { Ok(()) } + async fn delete_ids_from_write_collections( + &self, + alias: &str, + field: &str, + ids: &[String], + ) -> Result<()> { + for ids in ids.chunks(DELETE_FILTER_ID_BATCH_SIZE) { + let filter = format!("{field}:[{}]", ids.iter().join(", ")); + self.delete_from_write_collections(alias, &filter).await?; + } + Ok(()) + } + async fn delete_legacy_filtered_collections(&self) -> Result<()> { let alias = self.config.get_alias_name("projects_filtered"); let live = self.client.get_alias(&alias).await?; @@ -1265,101 +1277,94 @@ impl SearchBackend for Typesense { Ok(()) } - async fn index_documents( + async fn apply_update( &self, - documents: &[UploadSearchProject], + update: SearchIndexUpdate<'_>, ) -> eyre::Result<()> { - if documents.is_empty() { - return Ok(()); - } + let projects_alias = self.config.get_alias_name("projects"); + let versions_alias = self.config.get_alias_name("versions"); - let alias = self.config.get_alias_name("projects"); - let collections = self.existing_write_collections(&alias).await?; - debug!( - ?collections, - num_documents = documents.len(), - "Inserting into collections", - ); - self.import_document_batches(&collections, documents) + let removed_project_ids = update + .removed_projects + .iter() + .map(ToString::to_string) + .collect::>(); + if !removed_project_ids.is_empty() { + self.delete_ids_from_write_collections( + &versions_alias, + "project_id", + &removed_project_ids, + ) + .await?; + self.delete_ids_from_write_collections( + &projects_alias, + "project_id", + &removed_project_ids, + ) .await?; - - debug!("Done importing"); - Ok(()) - } - - async fn index_version_documents( - &self, - documents: &[UploadSearchVersion], - ) -> eyre::Result<()> { - if documents.is_empty() { - return Ok(()); } - let alias = self.config.get_alias_name("versions"); - let collections = self.existing_write_collections(&alias).await?; - debug!( - ?collections, - num_documents = documents.len(), - "Inserting version documents into collections", - ); - self.import_version_document_batches(&collections, documents) + let version_ids = update + .removed_versions + .iter() + .map(ToString::to_string) + .chain( + update + .versions + .iter() + .map(|document| document.version_id.clone()), + ) + .unique() + .collect::>(); + if !version_ids.is_empty() { + self.delete_ids_from_write_collections( + &versions_alias, + "id", + &version_ids, + ) .await?; - - debug!("Done importing version documents"); - Ok(()) - } - - async fn remove_project_documents( - &self, - ids: &[ProjectId], - ) -> eyre::Result<()> { - if ids.is_empty() { - return Ok(()); } - let id_list = ids + let project_ids = update + .projects .iter() - .map(|id| to_base62(id.0)) - .collect::>() - .join(", "); - let filter = format!("project_id:[{id_list}]"); - - let projects_alias = self.config.get_alias_name("projects"); - self.delete_from_write_collections(&projects_alias, &filter) + .map(|document| document.project_id.clone()) + .unique() + .collect::>(); + if !project_ids.is_empty() { + self.delete_ids_from_write_collections( + &projects_alias, + "project_id", + &project_ids, + ) .await?; - - debug!("Done"); - Ok(()) - } - - async fn remove_project_version_documents( - &self, - ids: &[ProjectId], - ) -> eyre::Result<()> { - if ids.is_empty() { - return Ok(()); } - let id_list = ids.iter().map(ToString::to_string).join(", "); - let filter = format!("project_id:[{id_list}]"); - let alias = self.config.get_alias_name("versions"); - self.delete_from_write_collections(&alias, &filter).await - } - - async fn remove_version_documents( - &self, - ids: &[VersionId], - ) -> eyre::Result<()> { - if ids.is_empty() { - return Ok(()); + if !update.projects.is_empty() { + let collections = + self.existing_write_collections(&projects_alias).await?; + debug!( + ?collections, + num_documents = update.projects.len(), + "Replacing project documents in collections", + ); + self.import_document_batches(&collections, update.projects) + .await?; } - let alias = self.config.get_alias_name("versions"); - for ids in ids.chunks(DELETE_FILTER_ID_BATCH_SIZE) { - let id_list = ids.iter().map(ToString::to_string).join(", "); - let filter = format!("id:[{id_list}]"); - self.delete_from_write_collections(&alias, &filter).await?; + if !update.versions.is_empty() { + let collections = + self.existing_write_collections(&versions_alias).await?; + debug!( + ?collections, + num_documents = update.versions.len(), + "Replacing version documents in collections", + ); + self.import_version_document_batches(&collections, update.versions) + .await?; } + + debug!("Done applying search index update"); Ok(()) } diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 94f5274169..d7c31af85b 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -17,7 +17,8 @@ use crate::{ env::ENV, models::ids::{ProjectId, VersionId}, search::{ - SearchBackend, + SearchBackend, SearchDocumentBatch, SearchIndexUpdate, + UploadSearchProject, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC, indexing::{index_project_documents, index_project_version_documents}, }, @@ -214,56 +215,24 @@ async fn consume_batch( project_ids_to_remove.len(), ); let start = Instant::now(); - - if !project_ids_to_remove.is_empty() { - let operation_start = Instant::now(); - info!( - project_count = project_ids_to_remove.len(), - "Removing project documents" - ); - search_backend - .remove_project_version_documents(&project_ids_to_remove) - .await - .wrap_err("failed to remove project version documents")?; - search_backend - .remove_project_documents(&project_ids_to_remove) - .await - .wrap_err("failed to remove project documents")?; - info!( - project_count = project_ids_to_remove.len(), - "Removed project documents in {:.2?}", - operation_start.elapsed() - ); - } - - if !version_ids_to_change.is_empty() { - let operation_start = Instant::now(); - search_backend - .remove_version_documents(&version_ids_to_change) - .await - .wrap_err("failed to remove changed version documents")?; - info!( - version_count = version_ids_to_change.len(), - "Removed changed version documents in {:.2?}", - operation_start.elapsed() - ); - } + let mut documents = SearchDocumentBatch::default(); if !project_ids_with_version_changes.is_empty() { let operation_start = Instant::now(); - reindex_changed_project_versions( + let changed_documents = build_changed_project_versions( ro_pool, redis_pool, - search_backend, &project_ids_with_version_changes, &version_ids_to_change, ) .await - .wrap_err("failed to reindex changed project versions")?; + .wrap_err("failed to build changed project versions")?; + documents.projects.extend(changed_documents.projects); + documents.versions.extend(changed_documents.versions); info!( project_count = project_ids_with_version_changes.len(), version_count = version_ids_to_change.len(), - "Reindexed changed project versions in {:.2?}", + "Built changed project versions in {:.2?}", operation_start.elapsed() ); } @@ -272,23 +241,39 @@ async fn consume_batch( let operation_start = Instant::now(); info!( project_count = project_ids_to_change.len(), - "Reindexing changed projects" + "Building changed projects" + ); + documents.projects.extend( + build_changed_projects(ro_pool, redis_pool, &project_ids_to_change) + .await + .wrap_err("failed to build changed projects")?, ); - reindex_projects( - ro_pool, - redis_pool, - search_backend, - &project_ids_to_change, - ) - .await - .wrap_err("failed to reindex changed project batch")?; info!( project_count = project_ids_to_change.len(), - "Reindexed changed projects in {:.2?}", + "Built changed projects in {:.2?}", operation_start.elapsed() ); } + let operation_start = Instant::now(); + search_backend + .apply_update(SearchIndexUpdate { + projects: &documents.projects, + versions: &documents.versions, + removed_projects: &project_ids_to_remove, + removed_versions: &version_ids_to_change, + }) + .await + .wrap_err("failed to apply search index update")?; + info!( + project_count = documents.projects.len(), + version_count = documents.versions.len(), + removed_project_count = project_ids_to_remove.len(), + removed_version_count = version_ids_to_change.len(), + "Applied search index update in {:.2?}", + operation_start.elapsed() + ); + for message in messages_to_commit { consumer .commit_message(&message, CommitMode::Async) @@ -320,22 +305,24 @@ pub async fn reindex_projects( search_backend: &dyn SearchBackend, project_ids: &[ProjectId], ) -> eyre::Result<()> { - info!("Removing documents for batch"); - search_backend.remove_project_documents(project_ids).await?; - info!("Creating project documents"); - index_changed_projects(ro_pool, redis_pool, search_backend, project_ids) + let projects = + build_changed_projects(ro_pool, redis_pool, project_ids).await?; + search_backend + .apply_update(SearchIndexUpdate { + projects: &projects, + ..SearchIndexUpdate::default() + }) .await?; Ok(()) } -async fn index_changed_projects( +async fn build_changed_projects( ro_pool: &PgPool, redis_pool: &RedisPool, - search_backend: &dyn SearchBackend, project_ids: &[ProjectId], -) -> eyre::Result<()> { +) -> eyre::Result> { let documents = index_project_documents(ro_pool, redis_pool, project_ids) .instrument(info_span!("index", batch_size = project_ids.len())) .await @@ -346,20 +333,16 @@ async fn index_changed_projects( ) })?; - info!("Fetched all project documents, indexing into backend"); - - search_backend.index_documents(&documents).await?; - - Ok(()) + info!("Fetched all project documents"); + Ok(documents) } -async fn reindex_changed_project_versions( +async fn build_changed_project_versions( ro_pool: &PgPool, redis_pool: &RedisPool, - search_backend: &dyn SearchBackend, project_ids: &[ProjectId], version_ids: &[VersionId], -) -> eyre::Result<()> { +) -> eyre::Result { let documents = index_project_version_documents( ro_pool, redis_pool, @@ -380,13 +363,31 @@ async fn reindex_changed_project_versions( ) })?; - search_backend.remove_project_documents(project_ids).await?; - search_backend.index_documents(&documents.projects).await?; - search_backend - .index_version_documents(&documents.versions) - .await?; + Ok(documents) +} - Ok(()) +pub async fn reindex_project_versions( + ro_pool: &PgPool, + redis_pool: &RedisPool, + search_backend: &dyn SearchBackend, + project_ids: &[ProjectId], + version_ids: &[VersionId], +) -> eyre::Result<()> { + let documents = build_changed_project_versions( + ro_pool, + redis_pool, + project_ids, + version_ids, + ) + .await?; + search_backend + .apply_update(SearchIndexUpdate { + projects: &documents.projects, + versions: &documents.versions, + removed_versions: version_ids, + ..SearchIndexUpdate::default() + }) + .await } #[derive(Debug, Deserialize)] diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 0b53f387ea..83e4fa433f 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -111,29 +111,9 @@ pub trait SearchBackend: Send + Sync { redis: RedisPool, ) -> eyre::Result<()>; - async fn index_documents( + async fn apply_update( &self, - documents: &[UploadSearchProject], - ) -> eyre::Result<()>; - - async fn index_version_documents( - &self, - documents: &[UploadSearchVersion], - ) -> eyre::Result<()>; - - async fn remove_project_documents( - &self, - ids: &[ProjectId], - ) -> eyre::Result<()>; - - async fn remove_project_version_documents( - &self, - ids: &[ProjectId], - ) -> eyre::Result<()>; - - async fn remove_version_documents( - &self, - ids: &[VersionId], + update: SearchIndexUpdate<'_>, ) -> eyre::Result<()>; async fn tasks(&self) -> eyre::Result; @@ -328,6 +308,17 @@ pub struct SearchDocumentBatch { pub versions: Vec, } +/// A logical search index mutation. Removals are applied before replacements, +/// so a document may be present in both a removed and replacement field. +#[derive(Debug, Clone, Copy, Default)] +pub struct SearchIndexUpdate<'a> { + pub projects: &'a [UploadSearchProject], + pub versions: &'a [UploadSearchVersion], + /// Projects and all of their version documents to remove. + pub removed_projects: &'a [ProjectId], + pub removed_versions: &'a [VersionId], +} + /// Nullable fields in Typesense-bound documents should use /// `skip_serializing_if = "Option::is_none"` so they are omitted instead of /// serialized as `null`. From 14ba540203881197cb711aa58af98febdbb2514c Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:22:12 +0100 Subject: [PATCH 21/26] cleanup pass --- apps/labrinth/src/routes/v3/projects.rs | 71 +++++-------------- .../src/routes/v3/version_creation.rs | 30 ++++---- apps/labrinth/src/routes/v3/versions.rs | 26 ++++--- 3 files changed, 47 insertions(+), 80 deletions(-) diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index 1e388aec5f..5da942c675 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -78,23 +78,6 @@ pub async fn clear_project_cache_and_queue_search( project_id: db_ids::DBProjectId, slug: Option, clear_dependencies: Option, -) -> Result<(), ApiError> { - clear_project_cache_and_queue_search_inner( - redis, - search_state, - project_id, - slug, - clear_dependencies, - ) - .await -} - -pub async fn clear_project_cache_and_queue_search_inner( - redis: &RedisPool, - search_state: &SearchState, - project_id: db_ids::DBProjectId, - slug: Option, - clear_dependencies: Option, ) -> Result<(), ApiError> { db_models::DBProject::clear_cache( project_id, @@ -109,30 +92,6 @@ pub async fn clear_project_cache_and_queue_search_inner( Ok(()) } -pub async fn clear_project_cache_and_queue_search_versions( - redis: &RedisPool, - search_state: &SearchState, - project_id: db_ids::DBProjectId, - slug: Option, - clear_dependencies: Option, - version_ids: impl IntoIterator, -) -> Result<(), ApiError> { - db_models::DBProject::clear_cache( - project_id, - slug, - clear_dependencies, - redis, - ) - .await?; - - search_state - .queue - .push_versions(project_id.into(), version_ids) - .await; - - Ok(()) -} - #[derive(Deserialize, Validate)] pub struct RandomProjects { #[validate(range(min = 1, max = 100))] @@ -1096,10 +1055,10 @@ pub async fn project_edit_internal( edit: Option>, mut component: &mut Option, perms: ProjectPermissions, - ) -> Result<(), ApiError> { + ) -> Result { let Some(edit) = edit else { // component is not specified in the input JSON - leave alone - return Ok(()); + return Ok(false); }; if !perms.contains(ProjectPermissions::EDIT_DETAILS) { @@ -1138,13 +1097,12 @@ pub async fn project_edit_internal( } } - Ok(()) + Ok(true) } - let reindex_version_project_types = - new_project.minecraft_java_server.is_some(); + let mut reindex_versions = false; - update( + reindex_versions |= update( &mut transaction, id, new_project.minecraft_server, @@ -1152,7 +1110,7 @@ pub async fn project_edit_internal( perms, ) .await?; - update( + reindex_versions |= update( &mut transaction, id, new_project.minecraft_java_server, @@ -1160,7 +1118,7 @@ pub async fn project_edit_internal( perms, ) .await?; - update( + reindex_versions |= update( &mut transaction, id, new_project.minecraft_bedrock_server, @@ -1213,16 +1171,21 @@ pub async fn project_edit_internal( transaction.commit().await?; - if reindex_version_project_types { - clear_project_cache_and_queue_search_versions( - &redis, - &search_state, + if reindex_versions { + db_models::DBProject::clear_cache( project_item.inner.id, project_item.inner.slug, None, - project_item.versions.iter().copied().map(VersionId::from), + &redis, ) .await?; + search_state + .queue + .push_versions( + project_item.inner.id.into(), + project_item.versions.iter().copied().map(VersionId::from), + ) + .await; } else { clear_project_cache_and_queue_search( &redis, diff --git a/apps/labrinth/src/routes/v3/version_creation.rs b/apps/labrinth/src/routes/v3/version_creation.rs index c32ae7a294..5e4a58be74 100644 --- a/apps/labrinth/src/routes/v3/version_creation.rs +++ b/apps/labrinth/src/routes/v3/version_creation.rs @@ -186,15 +186,12 @@ pub async fn version_create( } } else if let Ok((_, project_id, version_id)) = &result { transaction.commit().await?; - super::projects::clear_project_cache_and_queue_search_versions( - &redis, - &search_state, - *project_id, - None, - Some(true), - [VersionId::from(*version_id)], - ) - .await?; + models::DBProject::clear_cache(*project_id, None, Some(true), &redis) + .await?; + search_state + .queue + .push_versions((*project_id).into(), [VersionId::from(*version_id)]) + .await; } result.map(|(response, _, _)| response) @@ -684,15 +681,12 @@ pub async fn upload_file_to_version( } } else if let Ok((_, project_id)) = &result { transaction.commit().await?; - super::projects::clear_project_cache_and_queue_search_versions( - &redis, - &search_state, - *project_id, - None, - Some(true), - [version_id], - ) - .await?; + models::DBProject::clear_cache(*project_id, None, Some(true), &redis) + .await?; + search_state + .queue + .push_versions((*project_id).into(), [version_id]) + .await; } result.map(|(response, _)| response) diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index da5d04f6e2..66f64beb23 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -854,15 +854,20 @@ pub async fn version_edit_helper( transaction.commit().await?; database::models::DBVersion::clear_cache(&version_item, &redis) .await?; - super::projects::clear_project_cache_and_queue_search_versions( - &redis, - &search_state, + database::models::DBProject::clear_cache( version_item.inner.project_id, None, Some(true), - [VersionId::from(version_item.inner.id)], + &redis, ) .await?; + search_state + .queue + .push_versions( + version_item.inner.project_id.into(), + [VersionId::from(version_item.inner.id)], + ) + .await; Ok(HttpResponse::NoContent().body("")) } else { Err(ApiError::CustomAuthentication( @@ -1248,15 +1253,20 @@ pub async fn version_delete( transaction.commit().await?; - super::projects::clear_project_cache_and_queue_search_versions( - &redis, - &search_state, + database::models::DBProject::clear_cache( version.inner.project_id, None, Some(true), - [VersionId::from(version.inner.id)], + &redis, ) .await?; + search_state + .queue + .push_versions( + version.inner.project_id.into(), + [VersionId::from(version.inner.id)], + ) + .await; let project_id = version.inner.project_id.into(); let version_id = version.inner.id.into(); reindex_project_versions( From 7a54515f63e62fa34d839da61e59a5d8d7245e6e Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:38:22 +0100 Subject: [PATCH 22/26] cleanup pass 2 --- apps/labrinth/src/routes/v2/versions.rs | 4 +-- apps/labrinth/src/routes/v3/projects.rs | 12 +------- apps/labrinth/src/routes/v3/versions.rs | 28 ++----------------- .../src/search/incremental/consume.rs | 24 ---------------- apps/labrinth/tests/search.rs | 13 +-------- 5 files changed, 5 insertions(+), 76 deletions(-) diff --git a/apps/labrinth/src/routes/v2/versions.rs b/apps/labrinth/src/routes/v2/versions.rs index 30ae89ac3c..353bb450f2 100644 --- a/apps/labrinth/src/routes/v2/versions.rs +++ b/apps/labrinth/src/routes/v2/versions.rs @@ -11,7 +11,7 @@ use crate::models::projects::{ use crate::models::v2::projects::LegacyVersion; use crate::queue::session::AuthQueue; use crate::routes::{v2_reroute, v3}; -use crate::search::{SearchBackend, SearchState}; +use crate::search::SearchState; use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web}; use serde::{Deserialize, Serialize}; use validator::Validate; @@ -488,7 +488,6 @@ pub async fn version_delete( pool: web::Data, redis: web::Data, session_queue: web::Data, - search_backend: web::Data, search_state: web::Data, ) -> Result { // Returns NoContent, so we don't need to convert the response @@ -498,7 +497,6 @@ pub async fn version_delete( pool, redis, session_queue, - search_backend, search_state, ) .await diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index 5da942c675..f1a0e64129 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -29,8 +29,7 @@ use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::routes::internal::delphi; use crate::search::{ - SearchBackend, SearchIndexUpdate, SearchQuery, SearchRequest, - SearchResults, SearchState, + SearchBackend, SearchQuery, SearchRequest, SearchResults, SearchState, }; use crate::util::error::Context; use crate::util::img; @@ -2813,15 +2812,6 @@ pub async fn project_delete_internal( &redis, ) .await?; - let project_id = project.inner.id.into(); - search_state - .backend - .apply_update(SearchIndexUpdate { - removed_projects: std::slice::from_ref(&project_id), - ..SearchIndexUpdate::default() - }) - .await - .wrap_internal_err("failed to remove project from search index")?; search_state .queue .push_project_removal(project.inner.id.into()) diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index 66f64beb23..1d8f5e2663 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -27,9 +27,7 @@ use crate::models::teams::ProjectPermissions; use crate::queue::file_scan::get_files_missing_attribution; use crate::queue::session::AuthQueue; use crate::routes::internal::delphi; -use crate::search::incremental::consume::reindex_project_versions; -use crate::search::{SearchBackend, SearchState}; -use crate::util::error::Context; +use crate::search::SearchState; use crate::util::img; use crate::util::validate::validation_errors_to_string; use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web}; @@ -1136,19 +1134,9 @@ pub async fn version_delete_route( pool: web::Data, redis: web::Data, session_queue: web::Data, - search_backend: web::Data, search_state: web::Data, ) -> Result { - version_delete( - req, - info, - pool, - redis, - session_queue, - search_backend, - search_state, - ) - .await + version_delete(req, info, pool, redis, session_queue, search_state).await } pub async fn version_delete( @@ -1157,7 +1145,6 @@ pub async fn version_delete( pool: web::Data, redis: web::Data, session_queue: web::Data, - search_backend: web::Data, search_state: web::Data, ) -> Result { let user = get_user_from_headers( @@ -1267,17 +1254,6 @@ pub async fn version_delete( [VersionId::from(version.inner.id)], ) .await; - let project_id = version.inner.project_id.into(); - let version_id = version.inner.id.into(); - reindex_project_versions( - &pool, - &redis, - search_backend.as_ref(), - std::slice::from_ref(&project_id), - std::slice::from_ref(&version_id), - ) - .await - .wrap_internal_err("failed to update search index after version removal")?; if result.is_some() { Ok(HttpResponse::NoContent().body("")) } else { diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index d7c31af85b..925c932a05 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -366,30 +366,6 @@ async fn build_changed_project_versions( Ok(documents) } -pub async fn reindex_project_versions( - ro_pool: &PgPool, - redis_pool: &RedisPool, - search_backend: &dyn SearchBackend, - project_ids: &[ProjectId], - version_ids: &[VersionId], -) -> eyre::Result<()> { - let documents = build_changed_project_versions( - ro_pool, - redis_pool, - project_ids, - version_ids, - ) - .await?; - search_backend - .apply_update(SearchIndexUpdate { - projects: &documents.projects, - versions: &documents.versions, - removed_versions: version_ids, - ..SearchIndexUpdate::default() - }) - .await -} - #[derive(Debug, Deserialize)] #[serde(untagged)] enum SearchProjectIndexQueueEvent { diff --git a/apps/labrinth/tests/search.rs b/apps/labrinth/tests/search.rs index c009aa929c..f583d4a387 100644 --- a/apps/labrinth/tests/search.rs +++ b/apps/labrinth/tests/search.rs @@ -213,18 +213,7 @@ async fn index_swaps() { test_env.api.remove_project("alpha", USER_USER_PAT).await; assert_status!(&resp, StatusCode::NO_CONTENT); - // We should wait for deletions to be indexed - let projects = test_env - .api - .search_deserialized( - None, - Some(json!([["categories:fabric"]])), - USER_USER_PAT, - ) - .await; - assert_eq!(projects.total_hits, 0); - - // When we reindex, it should be still gone + // When we reindex, the deleted project should be gone let resp = test_env.api.reset_search_index().await; assert_status!(&resp, StatusCode::NO_CONTENT); From 5b5c96cf787a8a125249221d34a47d0a177b2224 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:00:39 +0100 Subject: [PATCH 23/26] standardise fn names --- apps/labrinth/src/background_task.rs | 2 +- apps/labrinth/src/queue/server_ping.rs | 5 +- apps/labrinth/src/routes/internal/admin.rs | 6 +- apps/labrinth/src/routes/v3/projects.rs | 7 +- .../src/routes/v3/version_creation.rs | 23 ++---- apps/labrinth/src/routes/v3/versions.rs | 4 +- .../src/search/backend/typesense/mod.rs | 2 +- apps/labrinth/src/search/incremental.rs | 4 +- .../src/search/incremental/consume.rs | 75 +++++++++---------- apps/labrinth/src/search/indexing.rs | 12 +-- apps/labrinth/src/search/mod.rs | 2 +- 11 files changed, 65 insertions(+), 77 deletions(-) diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index 0c3fb13ef0..b49692fa81 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -177,7 +177,7 @@ pub async fn index_search( search_backend: web::Data, ) -> eyre::Result<()> { info!("Indexing local database"); - search_backend.index_projects(ro_pool, redis_pool).await + search_backend.rebuild_index(ro_pool, redis_pool).await } pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> { diff --git a/apps/labrinth/src/queue/server_ping.rs b/apps/labrinth/src/queue/server_ping.rs index 5b6305961b..1941da8f40 100644 --- a/apps/labrinth/src/queue/server_ping.rs +++ b/apps/labrinth/src/queue/server_ping.rs @@ -181,8 +181,9 @@ impl ServerPingQueue { None, &self.redis, ); - let queue_search = - self.incremental_search_queue.push(*project_id); + let queue_search = self + .incremental_search_queue + .push_project_change(*project_id); let (clear_cache_result, _) = join(clear_cache, queue_search).await; diff --git a/apps/labrinth/src/routes/internal/admin.rs b/apps/labrinth/src/routes/internal/admin.rs index acd899e618..30905800f3 100644 --- a/apps/labrinth/src/routes/internal/admin.rs +++ b/apps/labrinth/src/routes/internal/admin.rs @@ -8,7 +8,7 @@ use crate::queue::analytics::AnalyticsQueue; use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::search::SearchBackend; -use crate::search::incremental::consume::reindex_project; +use crate::search::incremental::consume::reindex_project_document; use crate::util::date::get_current_tenths_of_ms; use crate::util::error::Context; use crate::util::guards::admin_key_guard; @@ -330,7 +330,7 @@ pub async fn force_reindex( ) -> Result { let redis = redis.get_ref(); search_backend - .index_projects(pool.as_ref().clone(), redis.clone()) + .rebuild_index(pool.as_ref().clone(), redis.clone()) .await .wrap_internal_err("failed to index projects")?; Ok(HttpResponse::NoContent().finish()) @@ -355,7 +355,7 @@ pub async fn force_reindex_project( search_backend: web::Data, ) -> Result { let (project_id,) = path.into_inner(); - reindex_project( + reindex_project_document( pool.as_ref(), redis.as_ref(), search_backend.as_ref(), diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index f1a0e64129..18afc646e6 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -86,7 +86,10 @@ pub async fn clear_project_cache_and_queue_search( ) .await?; - search_state.queue.push(project_id.into()).await; + search_state + .queue + .push_project_change(project_id.into()) + .await; Ok(()) } @@ -1180,7 +1183,7 @@ pub async fn project_edit_internal( .await?; search_state .queue - .push_versions( + .push_version_changes( project_item.inner.id.into(), project_item.versions.iter().copied().map(VersionId::from), ) diff --git a/apps/labrinth/src/routes/v3/version_creation.rs b/apps/labrinth/src/routes/v3/version_creation.rs index 5e4a58be74..dea4235d6f 100644 --- a/apps/labrinth/src/routes/v3/version_creation.rs +++ b/apps/labrinth/src/routes/v3/version_creation.rs @@ -20,8 +20,8 @@ use crate::models::notifications::NotificationBody; use crate::models::pack::PackFileHash; use crate::models::pats::Scopes; use crate::models::projects::{ - Dependency, FileType, Loader, ProjectStatus, Version, VersionFile, - VersionStatus, VersionType, + Dependency, FileType, Loader, Version, VersionFile, VersionStatus, + VersionType, }; use crate::models::projects::{DependencyType, skip_nulls}; use crate::models::teams::ProjectPermissions; @@ -190,7 +190,10 @@ pub async fn version_create( .await?; search_state .queue - .push_versions((*project_id).into(), [VersionId::from(*version_id)]) + .push_version_changes( + (*project_id).into(), + [VersionId::from(*version_id)], + ) .await; } @@ -567,18 +570,6 @@ async fn version_create_inner( } } - sqlx::query!( - " - UPDATE mods - SET queued = NOW() - WHERE id = $1 AND status = $2 - ", - project_id as models::DBProjectId, - ProjectStatus::Processing.as_str(), - ) - .execute(&mut *transaction) - .await?; - Ok(( HttpResponse::Ok().json(response), project_id, @@ -685,7 +676,7 @@ pub async fn upload_file_to_version( .await?; search_state .queue - .push_versions((*project_id).into(), [version_id]) + .push_version_changes((*project_id).into(), [version_id]) .await; } diff --git a/apps/labrinth/src/routes/v3/versions.rs b/apps/labrinth/src/routes/v3/versions.rs index 1d8f5e2663..55b922d7d9 100644 --- a/apps/labrinth/src/routes/v3/versions.rs +++ b/apps/labrinth/src/routes/v3/versions.rs @@ -861,7 +861,7 @@ pub async fn version_edit_helper( .await?; search_state .queue - .push_versions( + .push_version_changes( version_item.inner.project_id.into(), [VersionId::from(version_item.inner.id)], ) @@ -1249,7 +1249,7 @@ pub async fn version_delete( .await?; search_state .queue - .push_versions( + .push_version_changes( version.inner.project_id.into(), [VersionId::from(version.inner.id)], ) diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index 7222a4006f..f9e81023cb 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -1167,7 +1167,7 @@ impl SearchBackend for Typesense { }) } - async fn index_projects( + async fn rebuild_index( &self, ro_pool: PgPool, redis: RedisPool, diff --git a/apps/labrinth/src/search/incremental.rs b/apps/labrinth/src/search/incremental.rs index a418e7638c..f96c63f508 100644 --- a/apps/labrinth/src/search/incremental.rs +++ b/apps/labrinth/src/search/incremental.rs @@ -36,11 +36,11 @@ impl IncrementalSearchQueue { } } - pub async fn push(&self, project_id: ProjectId) { + pub async fn push_project_change(&self, project_id: ProjectId) { self.operations.lock().await.push_project_change(project_id); } - pub async fn push_versions( + pub async fn push_version_changes( &self, project_id: ProjectId, version_ids: impl IntoIterator, diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 925c932a05..f5ba505082 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -20,7 +20,7 @@ use crate::{ SearchBackend, SearchDocumentBatch, SearchIndexUpdate, UploadSearchProject, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC, - indexing::{index_project_documents, index_project_version_documents}, + indexing::{build_project_documents, build_version_change_documents}, }, util::kafka::{ INCREMENTAL_INDEX_SEARCH_TASK, KAFKA_OPERATION_INTERVAL, @@ -219,14 +219,25 @@ async fn consume_batch( if !project_ids_with_version_changes.is_empty() { let operation_start = Instant::now(); - let changed_documents = build_changed_project_versions( + let changed_documents = build_version_change_documents( ro_pool, redis_pool, &project_ids_with_version_changes, &version_ids_to_change, ) + .instrument(info_span!( + "index", + batch_size = project_ids_with_version_changes.len(), + version_count = version_ids_to_change.len() + )) .await - .wrap_err("failed to build changed project versions")?; + .wrap_err_with(|| { + format!( + "failed to build search documents for {} projects and {} versions", + project_ids_with_version_changes.len(), + version_ids_to_change.len() + ) + })?; documents.projects.extend(changed_documents.projects); documents.versions.extend(changed_documents.versions); info!( @@ -244,9 +255,13 @@ async fn consume_batch( "Building changed projects" ); documents.projects.extend( - build_changed_projects(ro_pool, redis_pool, &project_ids_to_change) - .await - .wrap_err("failed to build changed projects")?, + build_changed_project_documents( + ro_pool, + redis_pool, + &project_ids_to_change, + ) + .await + .wrap_err("failed to build changed projects")?, ); info!( project_count = project_ids_to_change.len(), @@ -290,16 +305,22 @@ async fn consume_batch( Ok(()) } -pub async fn reindex_project( +pub async fn reindex_project_document( ro_pool: &PgPool, redis_pool: &RedisPool, search_backend: &dyn SearchBackend, project_id: ProjectId, ) -> eyre::Result<()> { - reindex_projects(ro_pool, redis_pool, search_backend, &[project_id]).await + reindex_project_documents( + ro_pool, + redis_pool, + search_backend, + &[project_id], + ) + .await } -pub async fn reindex_projects( +pub async fn reindex_project_documents( ro_pool: &PgPool, redis_pool: &RedisPool, search_backend: &dyn SearchBackend, @@ -307,7 +328,8 @@ pub async fn reindex_projects( ) -> eyre::Result<()> { info!("Creating project documents"); let projects = - build_changed_projects(ro_pool, redis_pool, project_ids).await?; + build_changed_project_documents(ro_pool, redis_pool, project_ids) + .await?; search_backend .apply_update(SearchIndexUpdate { projects: &projects, @@ -318,12 +340,12 @@ pub async fn reindex_projects( Ok(()) } -async fn build_changed_projects( +async fn build_changed_project_documents( ro_pool: &PgPool, redis_pool: &RedisPool, project_ids: &[ProjectId], ) -> eyre::Result> { - let documents = index_project_documents(ro_pool, redis_pool, project_ids) + let documents = build_project_documents(ro_pool, redis_pool, project_ids) .instrument(info_span!("index", batch_size = project_ids.len())) .await .wrap_err_with(|| { @@ -337,35 +359,6 @@ async fn build_changed_projects( Ok(documents) } -async fn build_changed_project_versions( - ro_pool: &PgPool, - redis_pool: &RedisPool, - project_ids: &[ProjectId], - version_ids: &[VersionId], -) -> eyre::Result { - let documents = index_project_version_documents( - ro_pool, - redis_pool, - project_ids, - version_ids, - ) - .instrument(info_span!( - "index", - batch_size = project_ids.len(), - version_count = version_ids.len() - )) - .await - .wrap_err_with(|| { - format!( - "failed to build search documents for {} projects and {} versions", - project_ids.len(), - version_ids.len() - ) - })?; - - Ok(documents) -} - #[derive(Debug, Deserialize)] #[serde(untagged)] enum SearchProjectIndexQueueEvent { diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 3338b6e590..dc90da75bb 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -121,7 +121,7 @@ pub async fn index_local( Ok((documents, *largest)) } -pub async fn index_project_documents( +pub async fn build_project_documents( pool: &PgPool, redis: &RedisPool, project_ids: &[ProjectId], @@ -171,14 +171,14 @@ pub async fn index_project_documents( .projects) } -pub async fn index_project_version_documents( +pub async fn build_version_change_documents( pool: &PgPool, redis: &RedisPool, project_ids: &[ProjectId], version_ids: &[VersionId], ) -> eyre::Result { let projects = - index_project_document_batch(pool, redis, project_ids).await?; + build_search_document_batch(pool, redis, project_ids).await?; let version_ids = version_ids .iter() .map(ToString::to_string) @@ -193,7 +193,7 @@ pub async fn index_project_version_documents( }) } -async fn index_project_document_batch( +async fn build_search_document_batch( pool: &PgPool, redis: &RedisPool, project_ids: &[ProjectId], @@ -359,7 +359,7 @@ async fn build_search_documents( .await?; info!("Indexing local versions!"); - let mut versions = index_versions(pool, project_ids.clone()).await?; + let mut versions = load_project_versions(pool, project_ids.clone()).await?; info!("Indexing local org owners!"); @@ -780,7 +780,7 @@ struct PartialVersion { date_published: DateTime, } -async fn index_versions( +async fn load_project_versions( pool: &PgPool, project_ids: Vec, ) -> Result>> { diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 83e4fa433f..33bfe26f51 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -105,7 +105,7 @@ pub trait SearchBackend: Send + Sync { info: &SearchRequest, ) -> Result; - async fn index_projects( + async fn rebuild_index( &self, ro_pool: PgPool, redis: RedisPool, From b0559153ad1729f6c364ad794e5ae244f4aaa81e Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:42:02 +0100 Subject: [PATCH 24/26] cleanup pass --- ...fa6d4b6212a8788503d5a53cef0f0be8bb981.json | 15 - ...51f87424ee39aac49cae5575c0d3313fb7f24.json | 22 - apps/labrinth/src/background_task.rs | 2 +- .../src/search/backend/typesense/mod.rs | 166 ++---- .../src/search/incremental/consume.rs | 97 ++-- apps/labrinth/src/search/indexing.rs | 80 +-- scripts/.gitignore | 1 - scripts/convert-typesense-project-docs.py | 443 --------------- scripts/create-dummy-projects.py | 123 ---- scripts/import-typesense-jsonl.py | 536 ------------------ scripts/query-typesense.py | 105 ---- 11 files changed, 124 insertions(+), 1466 deletions(-) delete mode 100644 apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json delete mode 100644 apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json delete mode 100644 scripts/.gitignore delete mode 100755 scripts/convert-typesense-project-docs.py delete mode 100755 scripts/create-dummy-projects.py delete mode 100755 scripts/import-typesense-jsonl.py delete mode 100644 scripts/query-typesense.py diff --git a/apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json b/apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json deleted file mode 100644 index f05a642c48..0000000000 --- a/apps/labrinth/.sqlx/query-10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE mods\n SET queued = NOW()\n WHERE id = $1 AND status = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "10dd90694457e57fbb0ed390326fa6d4b6212a8788503d5a53cef0f0be8bb981" -} diff --git a/apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json b/apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json deleted file mode 100644 index 59ab74bde2..0000000000 --- a/apps/labrinth/.sqlx/query-8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT id FROM versions WHERE mod_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "8145d09f7c6c5d2e18a10ef814551f87424ee39aac49cae5575c0d3313fb7f24" -} diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index b49692fa81..a7e108af27 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -19,7 +19,7 @@ use crate::util::anrok; use actix_web::web; use clap::ValueEnum; use eyre::WrapErr; -use tracing::{info, instrument}; +use tracing::info; #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)] #[clap(rename_all = "kebab_case")] diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index f9e81023cb..b97b9ff07f 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -814,74 +814,16 @@ impl Typesense { .transpose() } - async fn ensure_collections(&self) -> Result<()> { - let projects_alias = self.config.get_alias_name("projects"); - let projects_collection = if let Some(collection) = - self.client.get_alias(&projects_alias).await? - { - collection - } else { - let collection = - self.config.get_next_collection_name(&projects_alias, false); - if !self.client.collection_exists(&collection).await? { - self.client - .create_collection(&Self::project_collection_schema( - &collection, - )) - .await?; - } - self.client - .upsert_alias(&projects_alias, &collection) - .await?; - collection - }; - - let versions_alias = self.config.get_alias_name("versions"); - if self.client.get_alias(&versions_alias).await?.is_none() { - let collection = - self.config.get_next_collection_name(&versions_alias, false); - if !self.client.collection_exists(&collection).await? { - self.client - .create_collection(&Self::version_collection_schema( - &collection, - &projects_collection, - )) - .await?; - } - self.client - .upsert_alias(&versions_alias, &collection) - .await?; - } - Ok(()) - } - - async fn delete_documents_by_filter_if_exists( - &self, - collection: &str, - filter: &str, - ) -> Result<()> { - if self.client.collection_exists(collection).await? { - self.client - .delete_documents_by_filter( - collection, - filter, - self.config.delete_batch_size, - ) - .await?; - } - - Ok(()) - } - - async fn import_document_batches( + async fn import_document_batches( &self, collections: &[String], - documents: &[UploadSearchProject], + documents: &[T], + serialize: fn(&[T]) -> Result, ) -> Result<()> { let batch_size = self.config.import_batch_size.max(1); for batch in documents.chunks(batch_size) { - let jsonl = documents_to_jsonl(batch)?; + let jsonl = serialize(batch)?; for collection in collections { info!( @@ -899,43 +841,21 @@ impl Typesense { Ok(()) } - async fn import_version_document_batches( - &self, - collections: &[String], - documents: &[UploadSearchVersion], - ) -> Result<()> { - let batch_size = self.config.import_batch_size.max(1); - - for batch in documents.chunks(batch_size) { - let jsonl = version_documents_to_jsonl(batch)?; - - for collection in collections { - info!( - collection, - document_count = batch.len(), - content_length_bytes = jsonl.len(), - "sending Typesense version document import" - ); - self.client - .import_documents(collection, jsonl.clone()) - .await?; - } - } - - Ok(()) - } - async fn existing_write_collections( &self, alias: &str, ) -> Result> { - let mut collections = Vec::new(); - - let live = self.client.get_alias(alias).await?; - let shadow_alt = self.config.get_next_collection_name(alias, true); - let shadow_current = self.config.get_next_collection_name(alias, false); + let mut collections = self + .client + .get_alias(alias) + .await? + .into_iter() + .collect_vec(); - for collection in live.into_iter().chain([shadow_alt, shadow_current]) { + for collection in [ + self.config.get_next_collection_name(alias, true), + self.config.get_next_collection_name(alias, false), + ] { if !collections.contains(&collection) && self.client.collection_exists(&collection).await? { @@ -946,27 +866,24 @@ impl Typesense { Ok(collections) } - async fn delete_from_write_collections( - &self, - alias: &str, - filter: &str, - ) -> Result<()> { - for collection in self.existing_write_collections(alias).await? { - self.delete_documents_by_filter_if_exists(&collection, filter) - .await?; - } - Ok(()) - } - async fn delete_ids_from_write_collections( &self, alias: &str, field: &str, ids: &[String], ) -> Result<()> { + let collections = self.existing_write_collections(alias).await?; for ids in ids.chunks(DELETE_FILTER_ID_BATCH_SIZE) { let filter = format!("{field}:[{}]", ids.iter().join(", ")); - self.delete_from_write_collections(alias, &filter).await?; + for collection in &collections { + self.client + .delete_documents_by_filter( + collection, + &filter, + self.config.delete_batch_size, + ) + .await?; + } } Ok(()) } @@ -1177,8 +1094,6 @@ impl SearchBackend for Typesense { let projects_alias = self.config.get_alias_name("projects"); let versions_alias = self.config.get_alias_name("versions"); - self.ensure_collections().await?; - let projects_current = self.client.get_alias(&projects_alias).await?; let versions_current = self.client.get_alias(&versions_alias).await?; @@ -1246,11 +1161,13 @@ impl SearchBackend for Typesense { self.import_document_batches( std::slice::from_ref(&projects_next), &documents.projects, + documents_to_jsonl, ) .await?; - self.import_version_document_batches( + self.import_document_batches( std::slice::from_ref(&versions_next), &documents.versions, + version_documents_to_jsonl, ) .await?; } @@ -1348,8 +1265,12 @@ impl SearchBackend for Typesense { num_documents = update.projects.len(), "Replacing project documents in collections", ); - self.import_document_batches(&collections, update.projects) - .await?; + self.import_document_batches( + &collections, + update.projects, + documents_to_jsonl, + ) + .await?; } if !update.versions.is_empty() { @@ -1360,8 +1281,12 @@ impl SearchBackend for Typesense { num_documents = update.versions.len(), "Replacing version documents in collections", ); - self.import_version_document_batches(&collections, update.versions) - .await?; + self.import_document_batches( + &collections, + update.versions, + version_documents_to_jsonl, + ) + .await?; } debug!("Done applying search index update"); @@ -1540,15 +1465,10 @@ fn rewrite_filter_for_join( } fn is_version_filter_field(field: &str) -> bool { - matches!( - field, - "categories" - | "project_types" - | "environment" - | "game_versions" - | "client_side" - | "server_side" - ) + ::iter().any(|search_field| { + search_field.is_version_field() + && search_field.typesense_spec().path == field + }) } fn is_negative_filter(expression: &str) -> bool { diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index f5ba505082..1af65b5a32 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -168,7 +168,14 @@ async fn consume_batch( } }; - match event.into_data() { + let event = match event { + SearchProjectIndexQueueEvent::Current(event) => event, + SearchProjectIndexQueueEvent::Legacy { project_id } => { + SearchProjectIndexQueueEventData::Change { project_id } + } + }; + + match event { SearchProjectIndexQueueEventData::Change { project_id } => { project_ids_to_change.insert(project_id); } @@ -200,7 +207,7 @@ async fn consume_batch( let project_ids_with_version_changes = project_ids_with_version_changes .into_iter() .collect::>(); - let project_ids_to_remove = + let mut project_ids_to_remove = project_ids_to_remove.into_iter().collect::>(); let version_ids_to_change = version_ids_to_change.into_iter().collect::>(); @@ -238,6 +245,10 @@ async fn consume_batch( version_ids_to_change.len() ) })?; + project_ids_to_remove.extend(missing_project_document_ids( + &project_ids_with_version_changes, + &changed_documents.projects, + )); documents.projects.extend(changed_documents.projects); documents.versions.extend(changed_documents.versions); info!( @@ -254,15 +265,27 @@ async fn consume_batch( project_count = project_ids_to_change.len(), "Building changed projects" ); - documents.projects.extend( - build_changed_project_documents( - ro_pool, - redis_pool, - &project_ids_to_change, + let changed_project_documents = build_project_documents( + ro_pool, + redis_pool, + &project_ids_to_change, + ) + .instrument(info_span!( + "index", + batch_size = project_ids_to_change.len() + )) + .await + .wrap_err_with(|| { + format!( + "failed to build search documents for {} projects", + project_ids_to_change.len() ) - .await - .wrap_err("failed to build changed projects")?, - ); + })?; + project_ids_to_remove.extend(missing_project_document_ids( + &project_ids_to_change, + &changed_project_documents, + )); + documents.projects.extend(changed_project_documents); info!( project_count = project_ids_to_change.len(), "Built changed projects in {:.2?}", @@ -327,12 +350,20 @@ pub async fn reindex_project_documents( project_ids: &[ProjectId], ) -> eyre::Result<()> { info!("Creating project documents"); - let projects = - build_changed_project_documents(ro_pool, redis_pool, project_ids) - .await?; + let projects = build_project_documents(ro_pool, redis_pool, project_ids) + .instrument(info_span!("index", batch_size = project_ids.len())) + .await + .wrap_err_with(|| { + format!( + "failed to build search documents for {} projects", + project_ids.len() + ) + })?; + let removed_projects = missing_project_document_ids(project_ids, &projects); search_backend .apply_update(SearchIndexUpdate { projects: &projects, + removed_projects: &removed_projects, ..SearchIndexUpdate::default() }) .await?; @@ -340,23 +371,22 @@ pub async fn reindex_project_documents( Ok(()) } -async fn build_changed_project_documents( - ro_pool: &PgPool, - redis_pool: &RedisPool, +fn missing_project_document_ids( project_ids: &[ProjectId], -) -> eyre::Result> { - let documents = build_project_documents(ro_pool, redis_pool, project_ids) - .instrument(info_span!("index", batch_size = project_ids.len())) - .await - .wrap_err_with(|| { - format!( - "failed to build search documents for {} projects", - project_ids.len() - ) - })?; + documents: &[UploadSearchProject], +) -> Vec { + let built_project_ids = documents + .iter() + .map(|project| project.project_id.as_str()) + .collect::>(); - info!("Fetched all project documents"); - Ok(documents) + project_ids + .iter() + .copied() + .filter(|project_id| { + !built_project_ids.contains(project_id.to_string().as_str()) + }) + .collect() } #[derive(Debug, Deserialize)] @@ -366,17 +396,6 @@ enum SearchProjectIndexQueueEvent { Legacy { project_id: ProjectId }, } -impl SearchProjectIndexQueueEvent { - fn into_data(self) -> SearchProjectIndexQueueEventData { - match self { - Self::Current(data) => data, - Self::Legacy { project_id } => { - SearchProjectIndexQueueEventData::Change { project_id } - } - } - } -} - #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "snake_case")] enum SearchProjectIndexQueueEventData { diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index dc90da75bb..d872a7a20c 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -117,7 +117,8 @@ pub async fn index_local( return Ok((SearchDocumentBatch::default(), i64::MAX)); }; - let documents = build_search_documents(pool, redis, db_projects).await?; + let documents = + build_search_documents(pool, redis, db_projects, None).await?; Ok((documents, *largest)) } @@ -126,49 +127,12 @@ pub async fn build_project_documents( redis: &RedisPool, project_ids: &[ProjectId], ) -> eyre::Result> { - let searchable_statuses = searchable_statuses(); - let project_ids = project_ids - .iter() - .map(|project_id| DBProjectId::from(*project_id).0) - .collect::>(); - - let db_projects = sqlx::query!( - r#" - SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, - m.icon_url icon_url, m.updated updated, m.approved approved, m.published, m.license license, m.slug slug, m.color, - m.components AS "components: sqlx::types::Json" - FROM mods m - WHERE m.status = ANY($1) AND m.id = ANY($2) - GROUP BY m.id - ORDER BY m.id ASC; - "#, - &searchable_statuses, - &project_ids, + let version_ids = HashSet::new(); + Ok( + build_search_document_batch(pool, redis, project_ids, &version_ids) + .await? + .projects, ) - .fetch(pool) - .map_ok(|m| PartialProject { - id: DBProjectId(m.id), - name: m.name, - summary: m.summary, - downloads: m.downloads, - follows: m.follows, - icon_url: m.icon_url, - updated: m.updated, - approved: m.approved.unwrap_or(m.published), - slug: m.slug, - color: m.color, - license: m.license, - components: m.components.0, - }) - .try_collect::>() - .await - .wrap_err("failed to fetch project")?; - - info!("Fetched partial projects"); - - Ok(build_search_documents(pool, redis, db_projects) - .await? - .projects) } pub async fn build_version_change_documents( @@ -177,26 +141,19 @@ pub async fn build_version_change_documents( project_ids: &[ProjectId], version_ids: &[VersionId], ) -> eyre::Result { - let projects = - build_search_document_batch(pool, redis, project_ids).await?; let version_ids = version_ids .iter() - .map(ToString::to_string) + .copied() + .map(DBVersionId::from) .collect::>(); - Ok(SearchDocumentBatch { - projects: projects.projects, - versions: projects - .versions - .into_iter() - .filter(|version| version_ids.contains(&version.version_id)) - .collect(), - }) + build_search_document_batch(pool, redis, project_ids, &version_ids).await } async fn build_search_document_batch( pool: &PgPool, redis: &RedisPool, project_ids: &[ProjectId], + version_ids: &HashSet, ) -> eyre::Result { let searchable_statuses = searchable_statuses(); let project_ids = project_ids @@ -236,13 +193,14 @@ async fn build_search_document_batch( .await .wrap_err("failed to fetch project")?; - build_search_documents(pool, redis, db_projects).await + build_search_documents(pool, redis, db_projects, Some(version_ids)).await } async fn build_search_documents( pool: &PgPool, redis: &RedisPool, db_projects: Vec, + version_ids: Option<&HashSet>, ) -> eyre::Result { let searchable_statuses = searchable_statuses(); let project_ids = db_projects.iter().map(|x| x.id.0).collect::>(); @@ -602,7 +560,13 @@ async fn build_search_documents( ); let project_id = ProjectId::from(project.id).to_string(); - version_uploads.extend(versions.iter().map(|version| { + version_uploads.extend(versions.iter().filter_map(|version| { + if version_ids.is_some_and(|version_ids| { + !version_ids.contains(&version.id) + }) { + return None; + } + let version_fields = VersionField::from_query_json( version.version_fields.clone(), &loader_field_definitions, @@ -662,7 +626,7 @@ async fn build_search_documents( ) }); - UploadSearchVersion { + Some(UploadSearchVersion { version_id: VersionId::from(version.id).to_string(), project_id: project_id.clone(), categories: version_categories, @@ -671,7 +635,7 @@ async fn build_search_documents( .date_published .timestamp(), loader_fields: fields, - } + }) })); let mut project_categories = categories; diff --git a/scripts/.gitignore b/scripts/.gitignore deleted file mode 100644 index bee8a64b79..0000000000 --- a/scripts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__ diff --git a/scripts/convert-typesense-project-docs.py b/scripts/convert-typesense-project-docs.py deleted file mode 100755 index 5cfd594028..0000000000 --- a/scripts/convert-typesense-project-docs.py +++ /dev/null @@ -1,443 +0,0 @@ -#!/usr/bin/env python3 -"""Split legacy Typesense JSONL into project and version documents.""" - -import argparse -import json -import os -import shutil -import sys -import tempfile -import time -import zlib -from collections import OrderedDict - - -VERSION_FILTER_PATHS = ( - "project_types", - "environment", - "game_versions", - "client_side", - "server_side", -) - -BASE62_DIGITS = { - character: index - for index, character in enumerate( - "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - ) -} - - -class ConversionError(Exception): - pass - - -def parse_args(): - parser = argparse.ArgumentParser( - description=( - "Convert a legacy Typesense JSONL export from one document per " - "version into separate project and version collections." - ) - ) - parser.add_argument("input", help="Legacy Typesense JSONL export") - parser.add_argument("projects_output", help="Destination project-document JSONL") - parser.add_argument("versions_output", help="Destination version-document JSONL") - parser.add_argument( - "--shards", - type=int, - default=512, - help="Temporary hash shards used to bound memory usage (default: 512)", - ) - parser.add_argument( - "--max-open-shards", - type=int, - default=64, - help="Maximum temporary shard files held open at once (default: 64)", - ) - parser.add_argument( - "--progress-interval", - type=float, - default=10.0, - help="Seconds between progress reports (default: 10)", - ) - parser.add_argument( - "--keep-temporary", - action="store_true", - help="Keep temporary shard files after completion or failure", - ) - return parser.parse_args() - - -def base62_value(value): - result = 0 - try: - for character in value: - result = result * 62 + BASE62_DIGITS[character] - except KeyError: - return -1 - return result - - -def get_path(document, path): - value = document - for segment in path.split("."): - if not isinstance(value, dict) or segment not in value: - return None, False - value = value[segment] - return value, True - - -def set_path(document, path, value): - segments = path.split(".") - current = document - for segment in segments[:-1]: - child = current.get(segment) - if not isinstance(child, dict): - child = {} - current[segment] = child - current = child - current[segments[-1]] = value - - -def unique_sorted(values): - return sorted(set(values)) - - -def extend_values(target, value): - if isinstance(value, list): - target.extend(value) - elif value is not None: - target.append(value) - - -def version_filter_document(document): - version_id = document.get("version_id") or document.get("id") - result = { - "id": str(version_id), - "version_id": str(version_id), - "project_id": str(document["project_id"]), - "version_published_timestamp": document.get( - "version_published_timestamp", -1 - ), - } - loaders = list(document.get("loaders") or []) - mrpack_loaders = list(document.get("mrpack_loaders") or []) - loaders.extend(mrpack_loaders) - if mrpack_loaders: - loaders = [loader for loader in loaders if loader != "mrpack"] - result["categories"] = unique_sorted(loaders) - for path in VERSION_FILTER_PATHS: - value, exists = get_path(document, path) - if exists and value is not None: - set_path(result, path, value) - return result - - -class ProjectAccumulator: - def __init__(self, document): - self.latest_document = document - self.latest_key = self.version_key(document) - self.versions = {} - self.categories = [] - self.version_categories = [] - self.loaders = [] - self.project_types = [] - self.client_side = [] - self.server_side = [] - self.add(document) - - @staticmethod - def version_key(document): - return ( - document.get("version_published_timestamp", -1), - base62_value(str(document.get("version_id", ""))), - ) - - def add(self, document): - project_id = document.get("project_id") - if project_id != self.latest_document.get("project_id"): - raise ConversionError("attempted to combine different projects") - - version_id = document.get("version_id") or document.get("id") - if not version_id: - raise ConversionError(f"project `{project_id}` has a version without an ID") - - key = self.version_key(document) - if key > self.latest_key: - self.latest_document = document - self.latest_key = key - - version_document = version_filter_document(document) - self.versions[str(version_id)] = (key, version_document) - extend_values(self.categories, document.get("categories")) - extend_values(self.version_categories, version_document.get("categories")) - extend_values(self.loaders, document.get("loaders")) - extend_values(self.project_types, document.get("project_types")) - extend_values(self.project_types, document.get("all_project_types")) - extend_values(self.client_side, document.get("client_side")) - extend_values(self.server_side, document.get("server_side")) - - def finish(self): - result = dict(self.latest_document) - project_id = str(result["project_id"]) - all_project_types = unique_sorted(self.project_types) - - result["id"] = project_id - result["version_id"] = str( - self.latest_document.get("version_id") - or self.latest_document.get("id") - ) - result["categories"] = unique_sorted(self.categories) - result["project_categories"] = unique_sorted( - set(self.categories) - set(self.version_categories) - ) - result["loaders"] = unique_sorted(self.loaders) - result["project_types"] = all_project_types - result["all_project_types"] = all_project_types - - project_loader_fields = result.get("project_loader_fields") - if not isinstance(project_loader_fields, dict): - project_loader_fields = {} - result["project_loader_fields"] = project_loader_fields - for field, value in project_loader_fields.items(): - result[field] = value - - if self.client_side: - result["client_side"] = unique_sorted(self.client_side) - if self.server_side: - result["server_side"] = unique_sorted(self.server_side) - - versions = [ - version - for _, version in sorted( - self.versions.values(), key=lambda item: item[0] - ) - ] - result.pop("versions", None) - return result, versions - - -class ShardWriter: - def __init__(self, directory, shard_count, max_open): - self.directory = directory - self.shard_count = shard_count - self.max_open = max_open - self.handles = OrderedDict() - - def path(self, shard): - return os.path.join(self.directory, f"shard-{shard:04d}.jsonl") - - def write(self, project_id, line): - shard = zlib.crc32(project_id.encode("utf-8")) % self.shard_count - handle = self.handles.pop(shard, None) - if handle is None: - if len(self.handles) >= self.max_open: - _, oldest = self.handles.popitem(last=False) - oldest.close() - handle = open(self.path(shard), "ab") - self.handles[shard] = handle - handle.write(line) - - def close(self): - for handle in self.handles.values(): - handle.close() - self.handles.clear() - - -def shard_input(args, temporary_directory): - writer = ShardWriter( - temporary_directory, - args.shards, - args.max_open_shards, - ) - input_size = os.path.getsize(args.input) - bytes_read = 0 - document_count = 0 - started_at = time.monotonic() - last_report = started_at - - try: - with open(args.input, "rb") as input_file: - for line_number, line in enumerate(input_file, start=1): - bytes_read += len(line) - if not line.strip(): - continue - try: - document = json.loads(line) - except json.JSONDecodeError as error: - raise ConversionError( - f"invalid JSON on input line {line_number}: {error}" - ) from error - project_id = document.get("project_id") - if not project_id: - raise ConversionError( - f"input line {line_number} has no `project_id`" - ) - writer.write(str(project_id), line) - document_count += 1 - - now = time.monotonic() - if now - last_report >= args.progress_interval: - percent = bytes_read / input_size * 100 if input_size else 100 - print( - f"sharding: {percent:.1f}% ({document_count:,} documents)", - flush=True, - ) - last_report = now - finally: - writer.close() - - print( - f"sharding complete: {document_count:,} version documents in " - f"{time.monotonic() - started_at:.1f}s", - flush=True, - ) - return document_count - - -def convert_shards( - args, - temporary_directory, - partial_projects_output, - partial_versions_output, -): - project_count = 0 - version_count = 0 - started_at = time.monotonic() - last_report = started_at - - with ( - open(partial_projects_output, "w", encoding="utf-8") as projects_file, - open(partial_versions_output, "w", encoding="utf-8") as versions_file, - ): - for shard in range(args.shards): - path = os.path.join(temporary_directory, f"shard-{shard:04d}.jsonl") - if not os.path.exists(path): - continue - - projects = {} - with open(path, "rb") as shard_file: - for line_number, line in enumerate(shard_file, start=1): - try: - document = json.loads(line) - except json.JSONDecodeError as error: - raise ConversionError( - f"invalid JSON in shard {shard}, line {line_number}: {error}" - ) from error - project_id = str(document["project_id"]) - if project_id in projects: - projects[project_id].add(document) - else: - projects[project_id] = ProjectAccumulator(document) - version_count += 1 - - for project_id in sorted(projects): - project, versions = projects[project_id].finish() - json.dump( - project, - projects_file, - separators=(",", ":"), - ensure_ascii=False, - ) - projects_file.write("\n") - for version in versions: - json.dump( - version, - versions_file, - separators=(",", ":"), - ensure_ascii=False, - ) - versions_file.write("\n") - project_count += 1 - - os.remove(path) - now = time.monotonic() - if now - last_report >= args.progress_interval: - print( - f"converting: shard {shard + 1}/{args.shards}, " - f"{project_count:,} projects", - flush=True, - ) - last_report = now - - print( - f"conversion complete: {version_count:,} versions into " - f"{project_count:,} projects in {time.monotonic() - started_at:.1f}s", - flush=True, - ) - return project_count, version_count - - -def main(): - args = parse_args() - if args.shards <= 0: - raise ConversionError("--shards must be greater than zero") - if args.max_open_shards <= 0: - raise ConversionError("--max-open-shards must be greater than zero") - if args.progress_interval <= 0: - raise ConversionError("--progress-interval must be greater than zero") - if not os.path.isfile(args.input): - raise ConversionError(f"input file does not exist: {args.input}") - paths = { - os.path.abspath(args.input), - os.path.abspath(args.projects_output), - os.path.abspath(args.versions_output), - } - if len(paths) != 3: - raise ConversionError("input and output paths must all differ") - - projects_directory = os.path.dirname(os.path.abspath(args.projects_output)) - versions_directory = os.path.dirname(os.path.abspath(args.versions_output)) - os.makedirs(projects_directory, exist_ok=True) - os.makedirs(versions_directory, exist_ok=True) - temporary_directory = tempfile.mkdtemp( - prefix="typesense-project-convert-", - dir=projects_directory, - ) - partial_projects_output = f"{args.projects_output}.partial" - partial_versions_output = f"{args.versions_output}.partial" - - try: - expected_versions = shard_input(args, temporary_directory) - project_count, version_count = convert_shards( - args, - temporary_directory, - partial_projects_output, - partial_versions_output, - ) - if version_count != expected_versions: - raise ConversionError( - f"sharded {expected_versions} versions but converted {version_count}" - ) - os.replace(partial_projects_output, args.projects_output) - os.replace(partial_versions_output, args.versions_output) - print( - f"wrote {project_count:,} project documents to " - f"{args.projects_output} and {version_count:,} version documents to " - f"{args.versions_output}", - flush=True, - ) - except Exception: - for partial_output in ( - partial_projects_output, - partial_versions_output, - ): - if os.path.exists(partial_output): - os.remove(partial_output) - raise - finally: - if args.keep_temporary: - print(f"temporary shards kept at {temporary_directory}", file=sys.stderr) - else: - shutil.rmtree(temporary_directory, ignore_errors=True) - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print("interrupted", file=sys.stderr) - sys.exit(130) - except (ConversionError, OSError) as error: - print(error, file=sys.stderr) - sys.exit(1) diff --git a/scripts/create-dummy-projects.py b/scripts/create-dummy-projects.py deleted file mode 100755 index c55ea25d2e..0000000000 --- a/scripts/create-dummy-projects.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import json -import time -import urllib.error -import urllib.request -from concurrent.futures import ThreadPoolExecutor, as_completed -from uuid import uuid4 - - -def make_body(boundary, slug, index): - data = { - "name": f"Dummy Load {index:04d}", - "slug": slug, - "summary": "A dummy project for local load testing.", - "description": "This project was generated locally for batch indexing tests.", - "initial_versions": [], - "is_draft": True, - "categories": [], - "license_id": "MIT", - } - - payload = json.dumps(data, separators=(",", ":")) - return ( - f"--{boundary}\r\n" - 'Content-Disposition: form-data; name="data"\r\n' - "Content-Type: application/json\r\n\r\n" - f"{payload}\r\n" - f"--{boundary}--\r\n" - ).encode() - - -def create_project(base_url, token, boundary, prefix, index, retries): - slug = f"{prefix}-{index:04d}" - body = make_body(boundary, slug, index) - headers = { - "Authorization": f"Bearer {token}", - "Content-Type": f"multipart/form-data; boundary={boundary}", - } - - for attempt in range(retries + 1): - req = urllib.request.Request( - f"{base_url}/v3/project", - data=body, - headers=headers, - method="POST", - ) - - try: - with urllib.request.urlopen(req, timeout=60) as resp: - resp.read() - return True, slug, resp.status, "" - except urllib.error.HTTPError as err: - text = err.read().decode("utf-8", errors="replace") - if err.code < 500 or attempt == retries: - return False, slug, err.code, text - except Exception as err: - if attempt == retries: - return False, slug, "error", repr(err) - - time.sleep(min(2**attempt, 10)) - - raise RuntimeError("unreachable") - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("--base-url", default="http://localhost:8000") - parser.add_argument("--token", default="mra_admin") - parser.add_argument("--count", type=int, default=1000) - parser.add_argument("--concurrency", type=int, default=2) - parser.add_argument("--retries", type=int, default=5) - args = parser.parse_args() - - boundary = "----modrinth-dummy-project-boundary" - prefix = f"dummy-load-{int(time.time())}-{uuid4().hex[:6]}" - - ok = 0 - failures = [] - - with ThreadPoolExecutor(max_workers=args.concurrency) as executor: - futures = [ - executor.submit( - create_project, - args.base_url, - args.token, - boundary, - prefix, - index, - args.retries, - ) - for index in range(args.count) - ] - - for completed, future in enumerate(as_completed(futures), 1): - success, slug, status, text = future.result() - if success: - ok += 1 - else: - failures.append((slug, status, text[:500])) - - if completed % 50 == 0: - print( - f"completed={completed} created={ok} failed={len(failures)}", - flush=True, - ) - - print( - json.dumps( - { - "prefix": prefix, - "attempted": args.count, - "created": ok, - "failed": len(failures), - "failures": failures[:20], - }, - indent=2, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/import-typesense-jsonl.py b/scripts/import-typesense-jsonl.py deleted file mode 100755 index 93b2fa03c7..0000000000 --- a/scripts/import-typesense-jsonl.py +++ /dev/null @@ -1,536 +0,0 @@ -#!/usr/bin/env python3 -""" -Import a JSONL Typesense export into local Labrinth Typesense collections. - -By default this imports tmp/export.jsonl into the Labrinth project search alias, -resolving the alias to its current backing collection first. - -Usage: - python3 scripts/import-typesense-jsonl.py - python3 scripts/import-typesense-jsonl.py tmp/export.jsonl --continue - python3 scripts/import-typesense-jsonl.py tmp/export.jsonl --alias labrinth_projects - python3 scripts/import-typesense-jsonl.py --collection labrinth_projects__alt -""" - -import argparse -import json -import os -import sys -import time -import urllib.error -import urllib.parse -import urllib.request - - -class TypesenseError(Exception): - pass - - -def default_aliases(): - prefix = os.environ.get("TYPESENSE_INDEX_PREFIX", "labrinth") - return [f"{prefix}_projects"] - - -def request_typesense(base_url, api_key, method, path, timeout, body=None, content_type=None): - url = f"{base_url.rstrip('/')}{path}" - request = urllib.request.Request(url, data=body, method=method) - request.add_header("X-TYPESENSE-API-KEY", api_key) - if content_type: - request.add_header("Content-Type", content_type) - - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return response.read() - except urllib.error.HTTPError as error: - error_body = error.read().decode("utf-8", errors="replace") - raise TypesenseError(f"{method} {path} failed ({error.code}): {error_body}") from error - except urllib.error.URLError as error: - raise TypesenseError(f"{method} {path} failed: {error.reason}") from error - - -def get_json_or_none(base_url, api_key, path, timeout): - try: - body = request_typesense(base_url, api_key, "GET", path, timeout) - except TypesenseError as error: - if " failed (404):" in str(error): - return None - raise - return json.loads(body.decode("utf-8")) - - -def request_json(base_url, api_key, method, path, timeout, body=None): - encoded_body = None - if body is not None: - encoded_body = json.dumps(body).encode("utf-8") - - response = request_typesense( - base_url, - api_key, - method, - path, - timeout, - body=encoded_body, - content_type="application/json" if body is not None else None, - ) - if not response: - return None - return json.loads(response.decode("utf-8")) - - -def resolve_alias_or_collection(base_url, api_key, name, timeout): - escaped = urllib.parse.quote(name, safe="") - alias = get_json_or_none(base_url, api_key, f"/aliases/{escaped}", timeout) - if alias is not None: - collection_name = alias.get("collection_name") - if not collection_name: - raise TypesenseError(f"Alias {name} did not include collection_name") - return collection_name - - collection = get_json_or_none(base_url, api_key, f"/collections/{escaped}", timeout) - if collection is None: - raise TypesenseError(f"No Typesense alias or collection exists for {name}") - return name - - -def collection_document_count(base_url, api_key, collection, timeout): - escaped = urllib.parse.quote(collection, safe="") - body = get_json_or_none(base_url, api_key, f"/collections/{escaped}", timeout) - if body is None: - raise TypesenseError(f"No Typesense collection exists for {collection}") - return int(body.get("num_documents", 0)) - - -def collection_schema_exists(base_url, api_key, collection, timeout): - escaped = urllib.parse.quote(collection, safe="") - return get_json_or_none(base_url, api_key, f"/collections/{escaped}", timeout) is not None - - -def collection_schema_for_create(name, source): - schema = { - "name": name, - "fields": source["fields"], - "default_sorting_field": source.get("default_sorting_field"), - "enable_nested_fields": source.get("enable_nested_fields", False), - "token_separators": source.get("token_separators", []), - "symbols_to_index": source.get("symbols_to_index", []), - } - return {key: value for key, value in schema.items() if value is not None} - - -def sibling_collection_name(collection): - if collection.endswith("__current"): - return f"{collection[:-len('__current')]}__alt" - if collection.endswith("__alt"): - return f"{collection[:-len('__alt')]}__current" - return None - - -def create_collection_from_source(base_url, api_key, collection, source_collection, timeout): - escaped_source = urllib.parse.quote(source_collection, safe="") - source = get_json_or_none(base_url, api_key, f"/collections/{escaped_source}", timeout) - if source is None: - raise TypesenseError( - f"Cannot create {collection}: source collection {source_collection} does not exist" - ) - - request_json( - base_url, - api_key, - "POST", - "/collections", - timeout, - body=collection_schema_for_create(collection, source), - ) - print(f"Created {collection} from schema of {source_collection}") - - -def ensure_collection_exists(collection, args): - if collection_schema_exists(args.typesense_url, args.api_key, collection, args.timeout): - return - - if not args.create_missing_collection: - raise TypesenseError(f"No Typesense collection exists for {collection}") - - source_collection = args.create_from_collection or sibling_collection_name(collection) - if not source_collection: - raise TypesenseError( - f"No Typesense collection exists for {collection}; pass --create-from-collection" - ) - - create_collection_from_source( - args.typesense_url, - args.api_key, - collection, - source_collection, - args.timeout, - ) - - -def typesense_health_ok(base_url, api_key, timeout): - try: - body = request_typesense(base_url, api_key, "GET", "/health", timeout) - except TypesenseError as error: - if " failed (503):" in str(error): - return False - raise - - return json.loads(body.decode("utf-8")).get("ok") is True - - -def wait_for_typesense_drain(collection, expected_documents, args): - started_at = time.monotonic() - last_reported_at = 0.0 - - while True: - document_count = collection_document_count( - args.typesense_url, args.api_key, collection, args.timeout - ) - healthy = typesense_health_ok(args.typesense_url, args.api_key, args.timeout) - - if document_count >= expected_documents and healthy: - elapsed = time.monotonic() - started_at - print( - f"{collection}: Typesense reports {document_count} document(s) " - f"and healthy after flush wait ({elapsed:.1f}s)" - ) - return elapsed - - elapsed = time.monotonic() - started_at - if elapsed > args.flush_timeout: - raise TypesenseError( - f"Timed out waiting for Typesense to flush {collection}: " - f"{document_count}/{expected_documents} document(s), " - f"health ok={healthy}" - ) - - if elapsed - last_reported_at >= args.flush_report_interval: - print( - f"{collection}: waiting for Typesense flush; " - f"{document_count}/{expected_documents} document(s), " - f"health ok={healthy} ({elapsed:.1f}s)" - ) - last_reported_at = elapsed - - time.sleep(args.flush_poll_interval) - - -def unique(values): - result = [] - seen = set() - for value in values: - if value in seen: - continue - seen.add(value) - result.append(value) - return result - - -def iter_jsonl_batches(path, batch_lines, max_batch_bytes, skip_documents=0): - batch = [] - batch_bytes = 0 - start_line = None - end_line = None - skipped = 0 - - with open(path, "rb") as export_file: - for line_number, line in enumerate(export_file, start=1): - if not line.strip(): - continue - - if skipped < skip_documents: - skipped += 1 - continue - - line_bytes = len(line) - should_flush = batch and ( - len(batch) >= batch_lines or batch_bytes + line_bytes > max_batch_bytes - ) - if should_flush: - yield start_line, end_line, len(batch), b"".join(batch) - batch = [] - batch_bytes = 0 - start_line = None - - if start_line is None: - start_line = line_number - end_line = line_number - batch.append(line) - batch_bytes += line_bytes - - if batch: - yield start_line, end_line, len(batch), b"".join(batch) - - -def count_jsonl_batches(path, batch_lines, max_batch_bytes, skip_documents=0): - batch_count = 0 - batch_lines_count = 0 - batch_bytes = 0 - skipped = 0 - - with open(path, "rb") as export_file: - for line in export_file: - if not line.strip(): - continue - - if skipped < skip_documents: - skipped += 1 - continue - - line_bytes = len(line) - if batch_lines_count and ( - batch_lines_count >= batch_lines or batch_bytes + line_bytes > max_batch_bytes - ): - batch_count += 1 - batch_lines_count = 0 - batch_bytes = 0 - - batch_lines_count += 1 - batch_bytes += line_bytes - - if batch_lines_count: - batch_count += 1 - - return batch_count - - -def parse_import_response(body, failures_to_print): - imported = 0 - failed = 0 - failures = [] - text = body.decode("utf-8", errors="replace") - - for line in text.splitlines(): - if not line: - continue - try: - result = json.loads(line) - except json.JSONDecodeError: - failed += 1 - if len(failures) < failures_to_print: - failures.append(line) - continue - - if result.get("success") is True: - imported += 1 - else: - failed += 1 - if len(failures) < failures_to_print: - failures.append(json.dumps(result, sort_keys=True)) - - if imported == 0 and failed == 0 and text.strip(): - failed = 1 - failures.append(text.strip()) - - return imported, failed, failures - - -def import_batch(base_url, api_key, collection, args, body): - query = urllib.parse.urlencode( - { - "action": args.action, - "dirty_values": args.dirty_values, - } - ) - escaped = urllib.parse.quote(collection, safe="") - if not body.endswith(b"\n"): - body += b"\n" - - return request_typesense( - base_url, - api_key, - "POST", - f"/collections/{escaped}/documents/import?{query}", - args.timeout, - body=body, - content_type="text/plain", - ) - - -def import_file(collection, args): - imported_total = 0 - batch_count = 0 - skip_documents = 0 - next_flush_at = args.flush_interval - started_at = time.monotonic() - flush_batch_started_at = started_at - flush_batch_start_total = 0 - - ensure_collection_exists(collection, args) - - if args.continue_import: - skip_documents = collection_document_count( - args.typesense_url, args.api_key, collection, args.timeout - ) - print(f"{collection}: continuing after {skip_documents} existing document(s)") - - total_batches = count_jsonl_batches( - args.file, args.batch_lines, args.max_batch_bytes, skip_documents - ) - if total_batches == 0: - print(f"{collection}: no remaining documents to import") - return 0 - - for start_line, end_line, document_count, body in iter_jsonl_batches( - args.file, args.batch_lines, args.max_batch_bytes, skip_documents - ): - batch_count += 1 - batch_started_at = time.monotonic() - response = import_batch(args.typesense_url, args.api_key, collection, args, body) - batch_elapsed = time.monotonic() - batch_started_at - imported, failed, failures = parse_import_response(response, args.failures_to_print) - - if failed: - print( - f"Typesense reported {failed} failed document(s) for " - f"{collection}, input lines {start_line}-{end_line}.", - file=sys.stderr, - ) - for failure in failures: - print(failure, file=sys.stderr) - raise TypesenseError(f"Import into {collection} failed") - - if imported != document_count: - raise TypesenseError( - f"Typesense acknowledged {imported} document(s) for " - f"{collection}, but the batch contained {document_count}" - ) - - imported_total += imported - elapsed = time.monotonic() - started_at - percent = batch_count / total_batches * 100 - print( - f"{collection}: batch {batch_count}/{total_batches} ({percent:.1f}%) " - f"imported {imported} document(s) " - f"from lines {start_line}-{end_line}; {imported_total} total " - f"(batch {batch_elapsed:.1f}s, elapsed {elapsed:.1f}s)" - ) - - if args.flush_interval and imported_total >= next_flush_at: - expected_documents = skip_documents + imported_total - flush_batch_elapsed = time.monotonic() - flush_batch_started_at - flush_batch_documents = imported_total - flush_batch_start_total - print( - f"{collection}: flush checkpoint at {imported_total} imported document(s); " - f"imported {flush_batch_documents} document(s) since previous flush " - f"in {flush_batch_elapsed:.1f}s" - ) - flush_wait_elapsed = wait_for_typesense_drain(collection, expected_documents, args) - print( - f"{collection}: flush checkpoint complete; " - f"import batch {flush_batch_elapsed:.1f}s, " - f"flush wait {flush_wait_elapsed:.1f}s" - ) - while imported_total >= next_flush_at: - next_flush_at += args.flush_interval - flush_batch_started_at = time.monotonic() - flush_batch_start_total = imported_total - - elapsed = time.monotonic() - started_at - print(f"{collection}: imported {imported_total} documents in {batch_count} batches ({elapsed:.1f}s)") - return imported_total - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Import a JSONL export into Typesense in payload-safe batches." - ) - parser.add_argument("jsonl_file", nargs="?", help="JSONL export to import") - parser.add_argument("--file", help="JSONL export to import") - parser.add_argument( - "--typesense-url", - default=os.environ.get("TYPESENSE_URL", "http://localhost:8108"), - help="Typesense URL", - ) - parser.add_argument( - "--api-key", - default=os.environ.get("TYPESENSE_API_KEY", "modrinth"), - help="Typesense API key", - ) - parser.add_argument( - "--alias", - action="append", - dest="aliases", - help="Typesense alias or collection name to import into. Repeat for multiple targets.", - ) - parser.add_argument( - "--collection", - action="append", - dest="collections", - help="Exact Typesense collection name. Repeat for multiple targets.", - ) - parser.add_argument("--action", default="create", choices=["create", "update", "upsert", "emplace"]) - parser.add_argument("--dirty-values", default="coerce_or_drop") - parser.add_argument("--batch-lines", type=int, default=5_000) - parser.add_argument("--max-batch-bytes", type=int, default=4 * 1024 * 1024) - parser.add_argument("--flush-interval", type=int, default=10_000) - parser.add_argument("--flush-timeout", type=float, default=900.0) - parser.add_argument("--flush-poll-interval", type=float, default=2.0) - parser.add_argument("--flush-report-interval", type=float, default=10.0) - parser.add_argument( - "--create-missing-collection", - action=argparse.BooleanOptionalAction, - default=True, - help="Create a missing __current/__alt collection by cloning its sibling schema.", - ) - parser.add_argument( - "--create-from-collection", - help="Collection whose schema should be cloned when creating a missing target collection.", - ) - parser.add_argument("--failures-to-print", type=int, default=20) - parser.add_argument("--timeout", type=float, default=120.0) - parser.add_argument( - "--continue", - action="store_true", - dest="continue_import", - help="Resume by skipping the number of documents already in the target collection.", - ) - return parser.parse_args() - - -def main(): - args = parse_args() - - if args.batch_lines <= 0: - raise TypesenseError("--batch-lines must be greater than zero") - if args.max_batch_bytes <= 0: - raise TypesenseError("--max-batch-bytes must be greater than zero") - if args.flush_interval < 0: - raise TypesenseError("--flush-interval must be greater than or equal to zero") - if args.flush_timeout <= 0: - raise TypesenseError("--flush-timeout must be greater than zero") - if args.flush_poll_interval <= 0: - raise TypesenseError("--flush-poll-interval must be greater than zero") - if args.flush_report_interval <= 0: - raise TypesenseError("--flush-report-interval must be greater than zero") - if args.aliases and args.collections: - raise TypesenseError("Use either --alias or --collection, not both") - if args.jsonl_file and args.file: - raise TypesenseError("Pass the JSONL path either positionally or with --file, not both") - args.file = args.jsonl_file or args.file or "tmp/export.jsonl" - if not os.path.isfile(args.file): - raise TypesenseError(f"File does not exist: {args.file}") - - if args.collections: - collections = unique(args.collections) - else: - aliases = unique(args.aliases or default_aliases()) - collections = [ - resolve_alias_or_collection(args.typesense_url, args.api_key, alias, args.timeout) - for alias in aliases - ] - collections = unique(collections) - - print(f"Importing {args.file} into {', '.join(collections)}") - for collection in collections: - import_file(collection, args) - - -if __name__ == "__main__": - try: - main() - except KeyboardInterrupt: - print("Interrupted.", file=sys.stderr) - sys.exit(130) - except TypesenseError as error: - print(error, file=sys.stderr) - sys.exit(1) diff --git a/scripts/query-typesense.py b/scripts/query-typesense.py deleted file mode 100644 index 24f151bf75..0000000000 --- a/scripts/query-typesense.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -""" -Query the local Labrinth Typesense collection directly. - -Usage: - python3 scripts/query-typesense.py sodium - python3 scripts/query-typesense.py foobarqux --per-page 10 -""" - -import argparse -import json -import os -import urllib.error -import urllib.request - - -def request_json(base_url, api_key, path, body, timeout): - request = urllib.request.Request( - f"{base_url.rstrip('/')}{path}", - data=json.dumps(body).encode("utf-8"), - method="POST", - headers={ - "X-TYPESENSE-API-KEY": api_key, - "Content-Type": "application/json", - }, - ) - - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return json.load(response) - except urllib.error.HTTPError as error: - error_body = error.read().decode("utf-8", errors="replace") - raise SystemExit(f"Typesense request failed ({error.code}): {error_body}") from error - except urllib.error.URLError as error: - raise SystemExit(f"Typesense request failed: {error.reason}") from error - - -def parse_args(): - parser = argparse.ArgumentParser(description="Query Typesense directly.") - parser.add_argument("query", help="Search query") - parser.add_argument( - "--typesense-url", - default=os.environ.get("TYPESENSE_URL", "http://localhost:8108"), - help="Typesense URL", - ) - parser.add_argument( - "--api-key", - default=os.environ.get("TYPESENSE_API_KEY", "modrinth"), - help="Typesense API key", - ) - parser.add_argument( - "--collection", - default="labrinth_projects__current", - help="Typesense collection to search", - ) - parser.add_argument("--per-page", type=int, default=5) - parser.add_argument("--timeout", type=float, default=30.0) - return parser.parse_args() - - -def main(): - args = parse_args() - payload = { - "searches": [ - { - "collection": args.collection, - "q": args.query, - "query_by": "name,summary,author,indexed_name,indexed_author", - "per_page": args.per_page, - "group_by": "project_id", - "group_limit": 1, - "facet_by": "project_id", - "max_facet_values": 0, - } - ] - } - - body = request_json( - args.typesense_url, args.api_key, "/multi_search", payload, args.timeout - ) - result = body["results"][0] - - print(f"query: {args.query}") - print(f"collection: {args.collection}") - print(f"found: {result.get('found', 0)}") - print(f"out_of: {result.get('out_of', 0)}") - print(f"search_time_ms: {result.get('search_time_ms', 0)}") - print() - - grouped_hits = result.get("grouped_hits", []) - if not grouped_hits: - print("no hits") - return - - for index, group in enumerate(grouped_hits, start=1): - hit = group["hits"][0] - document = hit["document"] - print(f"{index}. {document.get('name', '')} ({document.get('slug', '')})") - print(f" project_id: {document.get('project_id', '')}") - print(f" author: {document.get('author', '')}") - print(f" summary: {document.get('summary', '')}") - - -if __name__ == "__main__": - main() From 587c4054c84e427108810f24958e665f1f773fec Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:06:07 +0100 Subject: [PATCH 25/26] fix compile --- apps/labrinth/src/background_task.rs | 2 +- apps/labrinth/src/search/indexing.rs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index a7e108af27..b49692fa81 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -19,7 +19,7 @@ use crate::util::anrok; use actix_web::web; use clap::ValueEnum; use eyre::WrapErr; -use tracing::info; +use tracing::{info, instrument}; #[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)] #[clap(rename_all = "kebab_case")] diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index d872a7a20c..93ec7f16d0 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -589,6 +589,12 @@ async fn build_search_documents( &mut version_project_types, ); + // SPECIAL BEHAVIOUR + // Todo: revisit. + // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category. + // These were previously considered the loader, and in v2, the loader is a category for searching. + // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories. + // The loaders are kept in the project document's aggregated loader fields as well, so that no information is lost on retrieval. let mut version_categories = version.loaders.clone(); let mrpack_loaders = fields .get("mrpack_loaders") From bb1fb6f86f17994a1206f9e303c628f5c6bdf734 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:27:33 +0100 Subject: [PATCH 26/26] factor out filter rewriting --- .../backend/typesense/filter_rewrite.rs | 442 ++++++++++++++++++ .../src/search/backend/typesense/mod.rs | 439 +---------------- 2 files changed, 448 insertions(+), 433 deletions(-) create mode 100644 apps/labrinth/src/search/backend/typesense/filter_rewrite.rs diff --git a/apps/labrinth/src/search/backend/typesense/filter_rewrite.rs b/apps/labrinth/src/search/backend/typesense/filter_rewrite.rs new file mode 100644 index 0000000000..ca8f221ddc --- /dev/null +++ b/apps/labrinth/src/search/backend/typesense/filter_rewrite.rs @@ -0,0 +1,442 @@ +use std::sync::LazyLock; + +use eyre::{Result, WrapErr, eyre}; +use itertools::Itertools; +use regex::Regex; +use serde_json::Value; + +use crate::search::SearchField; + +#[derive(Clone, Default)] +struct JoinedFilterClause { + project: Vec, + version: Vec, +} + +pub(super) fn rewrite_filter_for_join( + filter: &str, + versions_collection: &str, +) -> Result { + const MAX_CLAUSES: usize = 256; + + fn parse(expression: &str) -> Result> { + let expression = trim_outer_parentheses(expression.trim()); + + let or_parts = split_top_level(expression, "||"); + if or_parts.len() > 1 { + let mut clauses = Vec::new(); + for part in or_parts { + clauses.extend(parse(part)?); + if clauses.len() > MAX_CLAUSES { + return Err(eyre!( + "search filter has too many boolean clauses" + )); + } + } + return Ok(clauses); + } + + let and_parts = split_top_level(expression, "&&"); + if and_parts.len() > 1 { + let mut clauses = vec![JoinedFilterClause::default()]; + for part in and_parts { + let right = parse(part)?; + if clauses.len().saturating_mul(right.len()) > MAX_CLAUSES { + return Err(eyre!( + "search filter has too many boolean clauses" + )); + } + clauses = clauses + .into_iter() + .cartesian_product(right) + .map(|(mut left, right)| { + left.project.extend(right.project); + left.version.extend(right.version); + left + }) + .collect(); + } + return Ok(clauses); + } + + let field = filter_field(expression).ok_or_else(|| { + eyre!("could not determine filter field in `{expression}`") + })?; + let mut clause = JoinedFilterClause::default(); + if field == "categories" { + let project_expression = + expression.replacen("categories", "project_categories", 1); + if is_negative_filter(expression) { + clause.project.push(project_expression); + clause.version.push(expression.to_string()); + Ok(vec![clause]) + } else { + Ok(vec![ + JoinedFilterClause { + project: vec![project_expression], + version: Vec::new(), + }, + JoinedFilterClause { + project: Vec::new(), + version: vec![expression.to_string()], + }, + ]) + } + } else { + if is_version_filter_field(field) { + clause.version.push(expression.to_string()); + } else { + clause.project.push(expression.to_string()); + } + Ok(vec![clause]) + } + } + + let clauses = parse(filter)?; + Ok(clauses + .into_iter() + .map(|clause| { + let mut parts = clause.project; + if !clause.version.is_empty() { + parts.push(format!( + "${versions_collection}({})", + clause.version.join(" && ") + )); + } + if parts.len() == 1 { + parts.pop().unwrap_or_default() + } else { + format!("({})", parts.join(" && ")) + } + }) + .join(" || ")) +} + +fn is_version_filter_field(field: &str) -> bool { + ::iter().any(|search_field| { + search_field.is_version_field() + && search_field.typesense_spec().path == field + }) +} + +fn is_negative_filter(expression: &str) -> bool { + expression + .split_once(':') + .is_some_and(|(_, value)| value.trim_start().starts_with("!=")) +} + +fn filter_field(expression: &str) -> Option<&str> { + let operator = expression.find(':')?; + let field = expression[..operator].trim(); + (!field.is_empty() + && field.chars().all(|character| { + character.is_ascii_alphanumeric() || "_.".contains(character) + })) + .then_some(field) +} + +fn trim_outer_parentheses(mut expression: &str) -> &str { + while expression.starts_with('(') + && expression.ends_with(')') + && matching_outer_parentheses(expression) + { + expression = expression[1..expression.len() - 1].trim(); + } + expression +} + +fn matching_outer_parentheses(expression: &str) -> bool { + let mut depth = 0; + let mut quote = None; + let mut escaped = false; + + for (index, character) in expression.char_indices() { + if escaped { + escaped = false; + continue; + } + if character == '\\' { + escaped = true; + continue; + } + if let Some(active_quote) = quote { + if character == active_quote { + quote = None; + } + continue; + } + if matches!(character, '\'' | '"' | '`') { + quote = Some(character); + continue; + } + match character { + '(' => depth += 1, + ')' => { + depth -= 1; + if depth == 0 && index + character.len_utf8() < expression.len() + { + return false; + } + } + _ => {} + } + } + + depth == 0 +} + +fn split_top_level<'a>(expression: &'a str, operator: &str) -> Vec<&'a str> { + let mut parts = Vec::new(); + let mut start = 0; + let mut parentheses = 0; + let mut brackets = 0; + let mut quote = None; + let mut escaped = false; + let bytes = expression.as_bytes(); + let mut index = 0; + + while index < bytes.len() { + let character = expression[index..].chars().next().unwrap_or_default(); + let width = character.len_utf8(); + if escaped { + escaped = false; + index += width; + continue; + } + if character == '\\' { + escaped = true; + index += width; + continue; + } + if let Some(active_quote) = quote { + if character == active_quote { + quote = None; + } + index += width; + continue; + } + if matches!(character, '\'' | '"' | '`') { + quote = Some(character); + index += width; + continue; + } + match character { + '(' => parentheses += 1, + ')' => parentheses -= 1, + '[' => brackets += 1, + ']' => brackets -= 1, + _ => {} + } + + if parentheses == 0 + && brackets == 0 + && expression[index..].starts_with(operator) + { + parts.push(expression[start..index].trim()); + index += operator.len(); + start = index; + continue; + } + index += width; + } + + if parts.is_empty() { + vec![expression] + } else { + parts.push(expression[start..].trim()); + parts + } +} + +/// Translates a Meilisearch filter expression into Typesense `filter_by` +/// syntax. +/// +/// Transformations (applied in order): +/// 1. `field (NOT )IN [v1, v2]` → `field:[v1, v2]` / `field:!=[v1, v2]` +/// 2. `field op value` for op ∈ {`!=`, `>=`, `<=`, `>`, `<`, `=`} +/// → `field:op value` +/// 3. `AND` / `OR` (case-insensitive) → `&&` / `||` +pub(super) fn meili_to_typesense(filter: &str) -> String { + static IN_RE: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?IN\s*\[([^\]]*)\]", + ) + .expect("valid regex") + }); + static EXISTS_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?EXISTS\b") + .expect("valid regex") + }); + static CMP_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"([a-zA-Z_.][a-zA-Z0-9_.]*)\s*(!=|>=|<=|>|<|=)\s*") + .expect("valid regex") + }); + static AND_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\bAND\b").expect("valid regex")); + static OR_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\bOR\b").expect("valid regex")); + + // Step 1 – IN / NOT IN + let s = IN_RE.replace_all(filter, |caps: ®ex::Captures<'_>| { + let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); + let is_not = caps.get(2).is_some(); + let values = caps.get(3).map(|m| m.as_str()).unwrap_or_default(); + if is_not { + format!("{field}:!=[{values}]") + } else { + format!("{field}:[{values}]") + } + }); + + let s = EXISTS_RE.replace_all(&s, |caps: ®ex::Captures<'_>| { + let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); + let is_not = caps.get(2).is_some(); + + match field { + "minecraft_java_server.ping.data" => format!( + "minecraft_java_server.is_online:= {}", + if is_not { "false" } else { "true" } + ), + _ => caps + .get(0) + .map(|m| m.as_str()) + .unwrap_or_default() + .to_string(), + } + }); + + // Step 2 – comparison operators (field op value → field:op value). + let s = CMP_RE.replace_all(&s, |caps: ®ex::Captures<'_>| { + let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); + let op = caps.get(2).map(|m| m.as_str()).unwrap_or_default(); + format!("{field}:{op} ") + }); + + // Step 3 – logical operators + let s = AND_RE.replace_all(&s, " && "); + let s = OR_RE.replace_all(&s, " || "); + s.into_owned() +} + +/// Converts the legacy Meilisearch `facets` JSON array into a Typesense +/// `filter_by` string. The outer array items are AND-ed together; the inner +/// array items are OR-ed together. +pub(super) fn facets_to_typesense(facets_json: &str) -> Result { + let facets = serde_json::from_str::>>(facets_json) + .wrap_err("failed to parse facets JSON")?; + + let and_parts: Vec = facets + .into_iter() + .map(|or_group| { + let or_parts: Vec = or_group + .into_iter() + .map(|facet| { + let conditions: Vec = if facet.is_array() { + serde_json::from_value::>(facet) + .unwrap_or_default() + } else { + vec![ + serde_json::from_value::(facet) + .unwrap_or_default(), + ] + }; + let and_conds: Vec = conditions + .into_iter() + .map(|condition| { + condition_to_typesense_filter(&condition) + }) + .collect(); + if and_conds.len() == 1 { + and_conds.into_iter().next().unwrap_or_default() + } else { + format!("({})", and_conds.join(" && ")) + } + }) + .collect(); + if or_parts.len() == 1 { + or_parts.into_iter().next().unwrap_or_default() + } else { + format!("({})", or_parts.join(" || ")) + } + }) + .collect(); + + Ok(and_parts.join(" && ")) +} + +/// Converts a single facet condition such as `"categories:mods"`, +/// `"categories=mods"`, or `"downloads!=100"` into a Typesense filter clause. +fn condition_to_typesense_filter(condition: &str) -> String { + // Match multi-character operators before their single-character prefixes, + // and range/inequality operators before the plain `=` equality arm. + for operator in ["!=", ">=", "<=", ">", "<"] { + if let Some((field, value)) = condition.split_once(operator) { + return format!("{}:{} {}", field.trim(), operator, value.trim()); + } + } + if let Some((field, value)) = condition.split_once(':') { + return format!("{}:= {}", field.trim(), value.trim()); + } + if let Some((field, value)) = condition.split_once('=') { + return format!("{}:= {}", field.trim(), value.trim()); + } + condition.to_string() +} + +#[cfg(test)] +mod tests { + use super::rewrite_filter_for_join; + + #[test] + fn project_filters_do_not_join_versions() { + assert_eq!( + rewrite_filter_for_join("license:= MIT", "versions").unwrap(), + "license:= MIT" + ); + } + + #[test] + fn correlated_version_filters_share_one_join() { + assert_eq!( + rewrite_filter_for_join( + "categories:= fabric && game_versions:= 1.21", + "versions", + ) + .unwrap(), + "(project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" + ); + } + + #[test] + fn project_and_version_filters_are_partitioned() { + assert_eq!( + rewrite_filter_for_join( + "license:= MIT && categories:= fabric", + "versions", + ) + .unwrap(), + "(license:= MIT && project_categories:= fabric) || (license:= MIT && $versions(categories:= fabric))" + ); + } + + #[test] + fn mixed_boolean_filters_preserve_version_correlation() { + assert_eq!( + rewrite_filter_for_join( + "(license:= MIT || categories:= fabric) && game_versions:= 1.21", + "versions", + ) + .unwrap(), + "(license:= MIT && $versions(game_versions:= 1.21)) || (project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" + ); + } + + #[test] + fn negative_categories_require_project_and_version_exclusion() { + assert_eq!( + rewrite_filter_for_join("categories:!= fabric", "versions") + .unwrap(), + "(project_categories:!= fabric && $versions(categories:!= fabric))" + ); + } +} diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index b97b9ff07f..28d628644c 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -3,7 +3,6 @@ use std::sync::LazyLock; use async_trait::async_trait; use eyre::{Result, eyre}; use itertools::Itertools; -use regex::Regex; use reqwest::Method; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -25,6 +24,12 @@ use crate::search::{ }; use crate::util::error::Context; +use self::filter_rewrite::{ + facets_to_typesense, meili_to_typesense, rewrite_filter_for_join, +}; + +mod filter_rewrite; + const DELETE_FILTER_ID_BATCH_SIZE: usize = 256; #[derive(Debug, Clone)] @@ -1358,435 +1363,3 @@ fn version_documents_to_jsonl( } Ok(out) } - -#[derive(Clone, Default)] -struct JoinedFilterClause { - project: Vec, - version: Vec, -} - -fn rewrite_filter_for_join( - filter: &str, - versions_collection: &str, -) -> Result { - const MAX_CLAUSES: usize = 256; - - fn parse(expression: &str) -> Result> { - let expression = trim_outer_parentheses(expression.trim()); - - let or_parts = split_top_level(expression, "||"); - if or_parts.len() > 1 { - let mut clauses = Vec::new(); - for part in or_parts { - clauses.extend(parse(part)?); - if clauses.len() > MAX_CLAUSES { - return Err(eyre!( - "search filter has too many boolean clauses" - )); - } - } - return Ok(clauses); - } - - let and_parts = split_top_level(expression, "&&"); - if and_parts.len() > 1 { - let mut clauses = vec![JoinedFilterClause::default()]; - for part in and_parts { - let right = parse(part)?; - if clauses.len().saturating_mul(right.len()) > MAX_CLAUSES { - return Err(eyre!( - "search filter has too many boolean clauses" - )); - } - clauses = clauses - .into_iter() - .cartesian_product(right) - .map(|(mut left, right)| { - left.project.extend(right.project); - left.version.extend(right.version); - left - }) - .collect(); - } - return Ok(clauses); - } - - let field = filter_field(expression).ok_or_else(|| { - eyre!("could not determine filter field in `{expression}`") - })?; - let mut clause = JoinedFilterClause::default(); - if field == "categories" { - let project_expression = - expression.replacen("categories", "project_categories", 1); - if is_negative_filter(expression) { - clause.project.push(project_expression); - clause.version.push(expression.to_string()); - Ok(vec![clause]) - } else { - Ok(vec![ - JoinedFilterClause { - project: vec![project_expression], - version: Vec::new(), - }, - JoinedFilterClause { - project: Vec::new(), - version: vec![expression.to_string()], - }, - ]) - } - } else { - if is_version_filter_field(field) { - clause.version.push(expression.to_string()); - } else { - clause.project.push(expression.to_string()); - } - Ok(vec![clause]) - } - } - - let clauses = parse(filter)?; - Ok(clauses - .into_iter() - .map(|clause| { - let mut parts = clause.project; - if !clause.version.is_empty() { - parts.push(format!( - "${versions_collection}({})", - clause.version.join(" && ") - )); - } - if parts.len() == 1 { - parts.pop().unwrap_or_default() - } else { - format!("({})", parts.join(" && ")) - } - }) - .join(" || ")) -} - -fn is_version_filter_field(field: &str) -> bool { - ::iter().any(|search_field| { - search_field.is_version_field() - && search_field.typesense_spec().path == field - }) -} - -fn is_negative_filter(expression: &str) -> bool { - expression - .split_once(':') - .is_some_and(|(_, value)| value.trim_start().starts_with("!=")) -} - -fn filter_field(expression: &str) -> Option<&str> { - let operator = expression.find(':')?; - let field = expression[..operator].trim(); - (!field.is_empty() - && field.chars().all(|character| { - character.is_ascii_alphanumeric() || "_.".contains(character) - })) - .then_some(field) -} - -fn trim_outer_parentheses(mut expression: &str) -> &str { - while expression.starts_with('(') - && expression.ends_with(')') - && matching_outer_parentheses(expression) - { - expression = expression[1..expression.len() - 1].trim(); - } - expression -} - -fn matching_outer_parentheses(expression: &str) -> bool { - let mut depth = 0; - let mut quote = None; - let mut escaped = false; - - for (index, character) in expression.char_indices() { - if escaped { - escaped = false; - continue; - } - if character == '\\' { - escaped = true; - continue; - } - if let Some(active_quote) = quote { - if character == active_quote { - quote = None; - } - continue; - } - if matches!(character, '\'' | '"' | '`') { - quote = Some(character); - continue; - } - match character { - '(' => depth += 1, - ')' => { - depth -= 1; - if depth == 0 && index + character.len_utf8() < expression.len() - { - return false; - } - } - _ => {} - } - } - - depth == 0 -} - -fn split_top_level<'a>(expression: &'a str, operator: &str) -> Vec<&'a str> { - let mut parts = Vec::new(); - let mut start = 0; - let mut parentheses = 0; - let mut brackets = 0; - let mut quote = None; - let mut escaped = false; - let bytes = expression.as_bytes(); - let mut index = 0; - - while index < bytes.len() { - let character = expression[index..].chars().next().unwrap_or_default(); - let width = character.len_utf8(); - if escaped { - escaped = false; - index += width; - continue; - } - if character == '\\' { - escaped = true; - index += width; - continue; - } - if let Some(active_quote) = quote { - if character == active_quote { - quote = None; - } - index += width; - continue; - } - if matches!(character, '\'' | '"' | '`') { - quote = Some(character); - index += width; - continue; - } - match character { - '(' => parentheses += 1, - ')' => parentheses -= 1, - '[' => brackets += 1, - ']' => brackets -= 1, - _ => {} - } - - if parentheses == 0 - && brackets == 0 - && expression[index..].starts_with(operator) - { - parts.push(expression[start..index].trim()); - index += operator.len(); - start = index; - continue; - } - index += width; - } - - if parts.is_empty() { - vec![expression] - } else { - parts.push(expression[start..].trim()); - parts - } -} - -/// Translates a Meilisearch filter expression into Typesense `filter_by` -/// syntax. -/// -/// Transformations (applied in order): -/// 1. `field (NOT )IN [v1, v2]` → `field:[v1, v2]` / `field:!=[v1, v2]` -/// 2. `field op value` for op ∈ {`!=`, `>=`, `<=`, `>`, `<`, `=`} -/// → `field:op value` -/// 3. `AND` / `OR` (case-insensitive) → `&&` / `||` -fn meili_to_typesense(filter: &str) -> String { - static IN_RE: LazyLock = LazyLock::new(|| { - Regex::new( - r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?IN\s*\[([^\]]*)\]", - ) - .expect("valid regex") - }); - static EXISTS_RE: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\b([a-zA-Z_.][a-zA-Z0-9_.]*)\s+(NOT\s+)?EXISTS\b") - .expect("valid regex") - }); - static CMP_RE: LazyLock = LazyLock::new(|| { - Regex::new(r"([a-zA-Z_.][a-zA-Z0-9_.]*)\s*(!=|>=|<=|>|<|=)\s*") - .expect("valid regex") - }); - static AND_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\bAND\b").expect("valid regex")); - static OR_RE: LazyLock = - LazyLock::new(|| Regex::new(r"(?i)\bOR\b").expect("valid regex")); - - // Step 1 – IN / NOT IN - let s = IN_RE.replace_all(filter, |caps: ®ex::Captures<'_>| { - let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); - let is_not = caps.get(2).is_some(); - let values = caps.get(3).map(|m| m.as_str()).unwrap_or_default(); - if is_not { - format!("{field}:!=[{values}]") - } else { - format!("{field}:[{values}]") - } - }); - - let s = EXISTS_RE.replace_all(&s, |caps: ®ex::Captures<'_>| { - let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); - let is_not = caps.get(2).is_some(); - - match field { - "minecraft_java_server.ping.data" => format!( - "minecraft_java_server.is_online:= {}", - if is_not { "false" } else { "true" } - ), - _ => caps - .get(0) - .map(|m| m.as_str()) - .unwrap_or_default() - .to_string(), - } - }); - - // Step 2 – comparison operators (field op value → field:op value). - let s = CMP_RE.replace_all(&s, |caps: ®ex::Captures<'_>| { - let field = caps.get(1).map(|m| m.as_str()).unwrap_or_default(); - let op = caps.get(2).map(|m| m.as_str()).unwrap_or_default(); - format!("{field}:{op} ") - }); - - // Step 3 – logical operators - let s = AND_RE.replace_all(&s, " && "); - let s = OR_RE.replace_all(&s, " || "); - s.into_owned() -} - -/// Converts the legacy Meilisearch `facets` JSON array into a Typesense -/// `filter_by` string. The outer array items are AND-ed together; the inner -/// array items are OR-ed together. -fn facets_to_typesense(facets_json: &str) -> Result { - let facets = serde_json::from_str::>>(facets_json) - .wrap_err("failed to parse facets JSON")?; - - let and_parts: Vec = facets - .into_iter() - .map(|or_group| { - let or_parts: Vec = or_group - .into_iter() - .map(|facet| { - let conditions: Vec = if facet.is_array() { - serde_json::from_value::>(facet) - .unwrap_or_default() - } else { - vec![ - serde_json::from_value::(facet) - .unwrap_or_default(), - ] - }; - let and_conds: Vec = conditions - .into_iter() - .map(|c| condition_to_typesense_filter(&c)) - .collect(); - if and_conds.len() == 1 { - and_conds.into_iter().next().unwrap_or_default() - } else { - format!("({})", and_conds.join(" && ")) - } - }) - .collect(); - if or_parts.len() == 1 { - or_parts.into_iter().next().unwrap_or_default() - } else { - format!("({})", or_parts.join(" || ")) - } - }) - .collect(); - - Ok(and_parts.join(" && ")) -} - -/// Converts a single facet condition such as `"categories:mods"`, -/// `"categories=mods"`, or `"downloads!=100"` into a Typesense filter clause. -fn condition_to_typesense_filter(cond: &str) -> String { - // Match multi-character operators before their single-character prefixes, - // and range/inequality operators before the plain `=` equality arm. - for op in ["!=", ">=", "<=", ">", "<"] { - if let Some((field, value)) = cond.split_once(op) { - return format!("{}:{} {}", field.trim(), op, value.trim()); - } - } - if let Some((field, value)) = cond.split_once(':') { - return format!("{}:= {}", field.trim(), value.trim()); - } - if let Some((field, value)) = cond.split_once('=') { - return format!("{}:= {}", field.trim(), value.trim()); - } - cond.to_string() -} - -#[cfg(test)] -mod tests { - use super::rewrite_filter_for_join; - - #[test] - fn project_filters_do_not_join_versions() { - assert_eq!( - rewrite_filter_for_join("license:= MIT", "versions").unwrap(), - "license:= MIT" - ); - } - - #[test] - fn correlated_version_filters_share_one_join() { - assert_eq!( - rewrite_filter_for_join( - "categories:= fabric && game_versions:= 1.21", - "versions", - ) - .unwrap(), - "(project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" - ); - } - - #[test] - fn project_and_version_filters_are_partitioned() { - assert_eq!( - rewrite_filter_for_join( - "license:= MIT && categories:= fabric", - "versions", - ) - .unwrap(), - "(license:= MIT && project_categories:= fabric) || (license:= MIT && $versions(categories:= fabric))" - ); - } - - #[test] - fn mixed_boolean_filters_preserve_version_correlation() { - assert_eq!( - rewrite_filter_for_join( - "(license:= MIT || categories:= fabric) && game_versions:= 1.21", - "versions", - ) - .unwrap(), - "(license:= MIT && $versions(game_versions:= 1.21)) || (project_categories:= fabric && $versions(game_versions:= 1.21)) || $versions(categories:= fabric && game_versions:= 1.21)" - ); - } - - #[test] - fn negative_categories_require_project_and_version_exclusion() { - assert_eq!( - rewrite_filter_for_join("categories:!= fabric", "versions") - .unwrap(), - "(project_categories:!= fabric && $versions(categories:!= fabric))" - ); - } -}