Use the sqlite module as the database layer

glean-core/src/core/mod.rs

@@ -15,7 +15,7 @@ use malloc_size_of_derive::MallocSizeOf;
 use once_cell::sync::OnceCell;
 use uuid::Uuid;
 
-use crate::database::Database;
+use crate::database::sqlite::Database;
 use crate::debug::DebugOptions;
 use crate::error::ClientIdFileError;
 use crate::event_database::EventDatabase;
@@ -341,7 +341,7 @@ impl Glean {
 
         {
             let data_store = glean.data_store.as_ref().unwrap();
-            let file_size = data_store.file_size.map(|n| n.get()).unwrap_or(0);
+            let file_size = data_store.file_size().map(|n| n.get()).unwrap_or(0);
 
             // If we have a client ID on disk, we check the database
             if let Some(stored_client_id) = stored_client_id {

glean-core/src/database/mod.rs

@@ -19,6 +19,8 @@ use crate::ErrorKind;
 use malloc_size_of::MallocSizeOf;
 use rkv::{StoreError, StoreOptions};
 
+pub mod sqlite;
+
 /// Unwrap a `Result`s `Ok` value or do the specified action.
 ///
 /// This is an alternative to the question-mark operator (`?`),

glean-core/src/database/sqlite.rs

@@ -6,7 +6,9 @@ use std::fs;
 use std::num::NonZeroU64;
 use std::path::Path;
 use std::str;
+use std::time::Duration;
 
+use malloc_size_of::MallocSizeOf;
 use rusqlite::params;
 use rusqlite::types::FromSqlError;
 use rusqlite::Transaction;
@@ -28,6 +30,13 @@ pub struct Database {
     conn: connection::Connection,
 }
 
+impl MallocSizeOf for Database {
+    fn size_of(&self, _ops: &mut malloc_size_of::MallocSizeOfOps) -> usize {
+        // FIXME: Can we get the allocated size of the connection?
+        0
+    }
+}
+
 impl std::fmt::Debug for Database {
     fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
         fmt.debug_struct("Database")
@@ -43,7 +52,12 @@ impl Database {
     ///
     /// This opens the underlying SQLite store and creates
     /// the underlying directory structure.
-    pub fn new(data_path: &Path, _delay_ping_lifetime_io: bool) -> Result<Self> {
+    pub fn new(
+        data_path: &Path,
+        _delay_ping_lifetime_io: bool,
+        _ping_lifetime_threshold: usize,
+        _ping_lifetime_max_time: Duration,
+    ) -> Result<Self> {
         let path = data_path.join("db");
         log::debug!("Database path: {:?}", path.display());
 
@@ -352,6 +366,10 @@ impl Database {
         })
     }
 
+    pub fn clear_lifetime_storage(&self, lifetime: Lifetime, storage_name: &str) -> Result<()> {
+        Ok(())
+    }
+
     /// Removes a single metric from the storage.
     ///
     /// # Arguments

glean-core/src/error.rs

@@ -65,6 +65,9 @@ pub enum ErrorKind {
 
     /// Parsing a UUID from a string failed
     UuidError(uuid::Error),
+
+    /// Database/SQLite error
+    SQLite(rusqlite::Error),
 }
 
 /// A specialized [`Error`] type for this crate's operations.
@@ -121,6 +124,7 @@ impl Display for Error {
                 s / 1024
             ),
             UuidError(e) => write!(f, "Failed to parse UUID: {}", e),
+            SQLite(e) => write!(f, "SQLite error: {}", e),
         }
     }
 }
@@ -155,6 +159,14 @@ impl From<serde_json::error::Error> for Error {
     }
 }
 
+impl From<rusqlite::Error> for Error {
+    fn from(error: rusqlite::Error) -> Error {
+        Error {
+            kind: ErrorKind::SQLite(error),
+        }
+    }
+}
+
 impl From<OsString> for Error {
     fn from(error: OsString) -> Error {
         Error {

glean-core/src/metrics/dual_labeled_counter.rs

@@ -424,7 +424,7 @@ fn get_seen_keys_and_categories(
     for store in &meta.inner.send_in_pings {
         glean
             .storage()
-            .iter_store_from(lifetime, store, Some(&prefix), &mut snapshotter);
+            .iter_store(lifetime, store, &mut snapshotter);
     }
 
     (seen_keys, seen_categories)

glean-core/src/metrics/labeled.rs

@@ -431,7 +431,7 @@ pub fn validate_dynamic_label(
     for store in &meta.inner.send_in_pings {
         glean
             .storage()
-            .iter_store_from(lifetime, store, Some(prefix), &mut snapshotter);
+            .iter_store(lifetime, store, &mut snapshotter);
     }
 
     let label_count = labels.len();

glean-core/src/ping/mod.rs

@@ -284,16 +284,6 @@ impl PingMaker {
         info!("Collecting {}", ping.name());
         let database = glean.storage();
 
-        // HACK: Only for metrics pings we add the ping timings.
-        // But we want that to persist until the next metrics ping is actually sent.
-        let write_samples = database.write_timings.replace(Vec::with_capacity(64));
-        if !write_samples.is_empty() {
-            glean
-                .database_metrics
-                .write_time
-                .accumulate_samples_sync(glean, &write_samples);
-        }
-
         let mut metrics_data = StorageManager.snapshot_as_json(database, ping.name(), true);
 
         let events_data = glean

glean-core/src/storage/mod.rs

@@ -10,7 +10,7 @@ use std::collections::HashMap;
 
 use serde_json::{json, Value as JsonValue};
 
-use crate::database::Database;
+use crate::database::sqlite::Database;
 use crate::metrics::dual_labeled_counter::RECORD_SEPARATOR;
 use crate::metrics::Metric;
 use crate::Lifetime;
@@ -132,13 +132,13 @@ impl StorageManager {
             }
         };
 
-        storage.iter_store_from(Lifetime::Ping, store_name, None, &mut snapshotter);
-        storage.iter_store_from(Lifetime::Application, store_name, None, &mut snapshotter);
-        storage.iter_store_from(Lifetime::User, store_name, None, &mut snapshotter);
+        storage.iter_store(Lifetime::Ping, store_name, &mut snapshotter);
+        storage.iter_store(Lifetime::Application, store_name, &mut snapshotter);
+        storage.iter_store(Lifetime::User, store_name, &mut snapshotter);
 
         // Add send in all pings client.annotations
         if store_name != "glean_client_info" {
-            storage.iter_store_from(Lifetime::Application, "all-pings", None, snapshotter);
+            storage.iter_store(Lifetime::Application, "all-pings", snapshotter);
         }
 
         if clear_store {
@@ -181,7 +181,7 @@ impl StorageManager {
             }
         };
 
-        storage.iter_store_from(metric_lifetime, store_name, None, &mut snapshotter);
+        storage.iter_store(metric_lifetime, store_name, &mut snapshotter);
 
         snapshot
     }
@@ -216,7 +216,7 @@ impl StorageManager {
             }
         };
 
-        storage.iter_store_from(metric_lifetime, store_name, None, &mut snapshotter);
+        storage.iter_store(metric_lifetime, store_name, &mut snapshotter);
 
         labels
     }
@@ -260,7 +260,7 @@ impl StorageManager {
             }
         };
 
-        storage.iter_store_from(Lifetime::Application, store_name, None, &mut snapshotter);
+        storage.iter_store(Lifetime::Application, store_name, &mut snapshotter);
 
         if snapshot.is_empty() {
             None