Text fastfields

Cargo.toml

@@ -9,6 +9,7 @@ members = [
   "nucliadb_protos/rust",
   "nucliadb_relations2",
   "nucliadb_texts2",
+  "nucliadb_texts3",
   "nucliadb_vectors",
   "vectors_benchmark",
 ]

nucliadb_models/src/nucliadb_models/internal/shards.py

@@ -33,6 +33,7 @@ class DocumentServiceEnum(str, Enum):
     DOCUMENT_V0 = "DOCUMENT_V0"
     DOCUMENT_V1 = "DOCUMENT_V1"
     DOCUMENT_V2 = "DOCUMENT_V2"
+    DOCUMENT_V3 = "DOCUMENT_V3"
 
 
 class ParagraphServiceEnum(str, Enum):

nucliadb_node/Cargo.toml

@@ -22,7 +22,7 @@ path = "src/bin/writer.rs"
 [dependencies]
 axum = "0.6"
 axum-server = "0.5"
-
+mrflagly = { version = "0.2.9", default-features = false }
 tonic = "0.11"
 tonic-health = "0.11"
 futures-core = "0.3.17"
@@ -68,6 +68,7 @@ rand = "0.8.4"
 nucliadb_core = { path = "../nucliadb_core" }
 nucliadb_procs = { path = "../nucliadb_procs" }
 nucliadb_texts2 = { path = "../nucliadb_texts2" }
+nucliadb_texts3 = { path = "../nucliadb_texts3" }
 nucliadb_paragraphs3 = { path = "../nucliadb_paragraphs3" }
 nucliadb_vectors = { path = "../nucliadb_vectors" }
 nucliadb_relations2 = { path = "../nucliadb_relations2" }

nucliadb_node/src/cache/writer_cache.rs

@@ -106,6 +106,7 @@ pub struct ShardWriterCache {
     pub shards_path: PathBuf,
     cache: Mutex<InnerCache>,
     metadata_manager: Arc<ShardsMetadataManager>,
+    settings: Settings,
 }
 
 impl ShardWriterCache {
@@ -114,6 +115,7 @@ impl ShardWriterCache {
             cache: Mutex::new(InnerCache::new(settings.max_open_shards)),
             shards_path: settings.shards_path(),
             metadata_manager: Arc::new(ShardsMetadataManager::new(settings.shards_path())),
+            settings,
         }
     }
 
@@ -123,7 +125,7 @@ impl ShardWriterCache {
 
     pub fn create(&self, new: NewShard) -> NodeResult<Arc<ShardWriter>> {
         let shard_id = new.shard_id.clone();
-        let (shard, metadata) = ShardWriter::new(new, &self.shards_path)?;
+        let (shard, metadata) = ShardWriter::new(new, &self.shards_path, &self.settings)?;
         let shard = Arc::new(shard);
 
         self.metadata_manager.add_metadata(metadata);

nucliadb_node/src/settings.rs

@@ -31,6 +31,7 @@
 //! providers (to parse from CLI for example).
 
 use anyhow::anyhow;
+use mrflagly::{FlagService, FlagServiceOptions};
 use nucliadb_core::tracing::Level;
 use object_store::ObjectStore;
 use serde::de::Unexpected;
@@ -89,22 +90,51 @@ pub fn load_settings() -> NodeResult<Settings> {
 const SENTRY_ENVS: [&str; 2] = ["stage", "prod"];
 const DEFAULT_ENV: &str = "stage";
 
+// Feature flags
+pub mod feature_flags {
+    pub const TEXTS3: &str = "nucliadb_node_texts3";
+}
+const DEFAULT_FEATURE_FLAGS: &str = r#"{"nucliadb_node_texts3": {"rollout": 100}}"#;
+
 #[derive(Clone)]
 pub struct Settings {
     env: Arc<EnvSettings>,
     pub object_store: Arc<dyn ObjectStore>,
+    pub flags: Arc<FlagService>,
 }
 
 impl From<EnvSettings> for Settings {
     fn from(value: EnvSettings) -> Self {
         let object_store = build_object_store_driver(&value);
+        let flags = Arc::new(build_flag_service(&value));
         Self {
             env: Arc::new(value),
             object_store,
+            flags,
         }
     }
 }
 
+fn build_flag_service(settings: &EnvSettings) -> FlagService {
+    if let Some(flag_settings_url) = &settings.flag_settings_url {
+        FlagService::new(FlagServiceOptions {
+            finder_type: mrflagly::FlagFinderType::URL,
+            url: Some(flag_settings_url.clone()),
+            data: None,
+            env_var: None,
+            refresh_interval: 300,
+        })
+    } else {
+        FlagService::new(FlagServiceOptions {
+            finder_type: mrflagly::FlagFinderType::JSON,
+            url: None,
+            data: Some(DEFAULT_FEATURE_FLAGS.to_string()),
+            env_var: None,
+            refresh_interval: 300,
+        })
+    }
+}
+
 pub fn build_object_store_driver(settings: &EnvSettings) -> Arc<dyn ObjectStore> {
     info!("File backend: {:?}", settings.file_backend);
     match settings.file_backend {
@@ -258,6 +288,9 @@ pub struct EnvSettings {
     pub s3_indexing_bucket: String,
     pub s3_endpoint: Option<String>,
     pub azure_account_url: Option<String>,
+
+    // Mr.Flagly
+    pub flag_settings_url: Option<String>,
 }
 
 impl EnvSettings {
@@ -340,6 +373,7 @@ impl Default for EnvSettings {
             s3_indexing_bucket: Default::default(),
             s3_endpoint: None,
             azure_account_url: Default::default(),
+            flag_settings_url: None,
         }
     }
 }

nucliadb_node/src/shards/shard_reader.rs

@@ -76,6 +76,7 @@ fn open_paragraphs_reader(version: u32, path: &Path) -> NodeResult<ParagraphsRea
 fn open_texts_reader(version: u32, path: &Path) -> NodeResult<TextsReaderPointer> {
     match version {
         2 => nucliadb_texts2::reader::TextReaderService::open(path).map(|i| Box::new(i) as TextsReaderPointer),
+        3 => nucliadb_texts3::reader::TextReaderService::open(path).map(|i| Box::new(i) as TextsReaderPointer),
         v => Err(node_error!("Invalid text reader version {v}")),
     }
 }
@@ -155,6 +156,7 @@ impl ShardReader {
             0 => DocumentService::DocumentV0,
             1 => DocumentService::DocumentV1,
             2 => DocumentService::DocumentV2,
+            3 => DocumentService::DocumentV3,
             i => panic!("Unknown document version {i}"),
         }
     }