test telemetry

.gitignore

@@ -7,3 +7,7 @@ web/dist
 web/tmp
 adapters
 plugins
+!qwdata/.gitkeep
+qwdata
+!grafana-storage/plugins
+grafana-storage

Cargo.lock

@@ -239,6 +239,20 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "actix-web-opentelemetry"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aaa592a5b9b3d96101434bca1024c6da6c23630163aec485428e613cd7100dcf"
+dependencies = [
+ "actix-http",
+ "actix-web",
+ "futures-util",
+ "opentelemetry 0.18.0",
+ "opentelemetry-semantic-conventions 0.10.0",
+ "serde",
+]
+
 [[package]]
 name = "actix_derive"
 version = "0.6.0"
@@ -3675,6 +3689,19 @@ dependencies = [
  "opentelemetry_sdk 0.19.0",
 ]
 
+[[package]]
+name = "opentelemetry-http"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a819b71d6530c4297b49b3cae2939ab3a8cc1b9f382826a1bc29dd0ca3864906"
+dependencies = [
+ "async-trait",
+ "bytes 1.4.0",
+ "http",
+ "opentelemetry_api 0.19.0",
+ "reqwest 0.11.18",
+]
+
 [[package]]
 name = "opentelemetry-otlp"
 version = "0.12.0"
@@ -3686,8 +3713,10 @@ dependencies = [
  "futures-util",
  "http",
  "opentelemetry 0.19.0",
+ "opentelemetry-http",
  "opentelemetry-proto",
  "prost",
+ "reqwest 0.11.18",
  "serde",
  "thiserror",
  "tokio 1.28.2",
@@ -3707,6 +3736,24 @@ dependencies = [
  "tonic",
 ]
 
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb"
+dependencies = [
+ "opentelemetry 0.18.0",
+]
+
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24e33428e6bf08c6f7fcea4ddb8e358fab0fe48ab877a87c70c6ebe20f673ce5"
+dependencies = [
+ "opentelemetry 0.19.0",
+]
+
 [[package]]
 name = "opentelemetry_api"
 version = "0.18.0"
@@ -3754,6 +3801,8 @@ dependencies = [
  "percent-encoding",
  "rand 0.8.5",
  "thiserror",
+ "tokio 1.28.2",
+ "tokio-stream",
 ]
 
 [[package]]
@@ -3774,6 +3823,8 @@ dependencies = [
  "percent-encoding",
  "rand 0.8.5",
  "thiserror",
+ "tokio 1.28.2",
+ "tokio-stream",
 ]
 
 [[package]]
@@ -3990,7 +4041,7 @@ dependencies = [
  "llvmenv",
  "model",
  "payment-adapter",
- "reqwest 0.11.17",
+ "reqwest 0.11.18",
  "rumqttc",
  "serde",
  "sqlx",
@@ -4121,19 +4172,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "plugin-telemetry"
-version = "0.1.0"
-dependencies = [
- "opentelemetry 0.19.0",
- "opentelemetry-otlp",
- "plugin-api",
- "serde",
- "tokio 1.28.2",
- "tracing",
- "tracing-bunyan-formatter",
-]
-
 [[package]]
 name = "polling"
 version = "2.8.0"
@@ -4693,13 +4731,14 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.17"
+version = "0.11.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91"
+checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
 dependencies = [
  "base64 0.21.0",
  "bytes 1.4.0",
  "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "h2 0.3.18",
@@ -4715,11 +4754,13 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite 0.2.9",
+ "rustls 0.21.2",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio 1.28.2",
  "tokio-native-tls",
+ "tokio-socks",
  "tokio-util 0.7.8",
  "tower-service",
  "url",
@@ -4858,7 +4899,7 @@ dependencies = [
  "minidom",
  "percent-encoding",
  "quick-xml",
- "reqwest 0.11.17",
+ "reqwest 0.11.18",
  "serde",
  "serde_derive",
  "sha2",
@@ -4950,6 +4991,18 @@ dependencies = [
  "webpki",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f"
+dependencies = [
+ "log",
+ "ring",
+ "rustls-webpki",
+ "sct",
+]
+
 [[package]]
 name = "rustls-native-certs"
 version = "0.6.2"
@@ -4971,6 +5024,16 @@ dependencies = [
  "base64 0.21.0",
 ]
 
+[[package]]
+name = "rustls-webpki"
+version = "0.100.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.12"
@@ -5264,7 +5327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd6ba8c048dc4beab827cebc1d54f1007f5ebdfa240812ed6faa89452fd34ba0"
 dependencies = [
  "data-encoding",
- "reqwest 0.11.17",
+ "reqwest 0.11.18",
  "serde",
  "serde_json",
  "thiserror",
@@ -5293,9 +5356,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.96"
+version = "1.0.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
+checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a"
 dependencies = [
  "itoa 1.0.6",
  "ryu",
@@ -5605,7 +5668,7 @@ dependencies = [
  "percent-encoding",
  "rand 0.8.5",
  "rust_decimal",
- "rustls",
+ "rustls 0.20.8",
  "rustls-pemfile",
  "serde",
  "serde_json",
@@ -5849,7 +5912,7 @@ dependencies = [
  "tokio-serde",
  "tokio-util 0.7.8",
  "tracing",
- "tracing-opentelemetry",
+ "tracing-opentelemetry 0.18.0",
 ]
 
 [[package]]
@@ -5863,6 +5926,23 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "telemetry-plugin"
+version = "0.1.0"
+dependencies = [
+ "async-trait",
+ "opentelemetry 0.19.0",
+ "opentelemetry-otlp",
+ "opentelemetry-semantic-conventions 0.11.0",
+ "plugin-api",
+ "serde",
+ "tokio 1.28.2",
+ "tracing",
+ "tracing-bunyan-formatter",
+ "tracing-opentelemetry 0.19.0",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.5.0"
@@ -6124,7 +6204,7 @@ version = "0.23.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
 dependencies = [
- "rustls",
+ "rustls 0.20.8",
  "tokio 1.28.2",
  "webpki",
 ]
@@ -6145,6 +6225,18 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "tokio-socks"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0"
+dependencies = [
+ "either",
+ "futures-util",
+ "thiserror",
+ "tokio 1.28.2",
+]
+
 [[package]]
 name = "tokio-stream"
 version = "0.1.14"
@@ -6388,6 +6480,21 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "tracing-opentelemetry"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00a39dcf9bfc1742fa4d6215253b33a6e474be78275884c216fc2a06267b3600"
+dependencies = [
+ "async-trait",
+ "once_cell",
+ "opentelemetry 0.19.0",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.17"
@@ -7110,6 +7217,26 @@ dependencies = [
  "wast 57.0.0",
 ]
 
+[[package]]
+name = "web-api-plugin-tester"
+version = "0.1.0"
+dependencies = [
+ "actix",
+ "actix-rt",
+ "actix-web",
+ "actix-web-opentelemetry",
+ "anyhow",
+ "futures 0.3.28",
+ "plugin-api",
+ "rand 0.8.5",
+ "reqwest 0.11.18",
+ "serde",
+ "serde_json",
+ "telemetry-plugin",
+ "toml 0.7.4",
+ "tracing",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.61"

Cargo.toml

@@ -28,7 +28,7 @@ members = [
     #    "vendor/t_pay",
     #    "vendor/pay_u",
     # PLUGINS
-    "crates/plugin-telemetry",
+    "crates/telemetry-plugin",
     "crates/plugin-api",
     # PAYMENT
     "crates/payment-adapter",
@@ -48,6 +48,9 @@ members = [
     "crates/file-storage-s3-plugin",
     # database
     "migration",
+
+    ###### TEST
+    "crates/web-api-plugin-tester",
 ]
 exclude = [
     "crates/web"

crates/config/src/lib.rs

@@ -5,6 +5,7 @@ use std::sync::Arc;
 use parking_lot::Mutex;
 use password_hash::SaltString;
 use serde::{Deserialize, Serialize};
+use toml::Value;
 
 #[derive(Debug, thiserror::Error)]
 pub enum Error {}
@@ -775,6 +776,10 @@ impl AppConfig {
         &mut self.web
     }
 
+    pub fn plugin_mut(&mut self) -> &mut HashMap<String, Value> {
+        &mut self.plugin
+    }
+
     pub fn plugin_config(&self, name: &str) -> Option<PluginConfig> {
         self.plugin.get(name).cloned().map(PluginConfig)
     }

crates/model/src/v4/mod.rs

@@ -1,103 +1,103 @@
-//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
+// //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
-
+//
-pub mod prelude;
+// pub mod prelude;
-
+//
-pub mod addresses;
+// pub mod addresses;
-pub mod analytics_configs;
+// pub mod analytics_configs;
-pub mod batch_jobs;
+// pub mod batch_jobs;
-pub mod cart_discounts;
+// pub mod cart_discounts;
-pub mod cart_gift_cards;
+// pub mod cart_gift_cards;
-pub mod carts;
+// pub mod carts;
-pub mod claim_images;
+// pub mod claim_images;
-pub mod claim_item_tags;
+// pub mod claim_item_tags;
-pub mod claim_items;
+// pub mod claim_items;
-pub mod claim_orders;
+// pub mod claim_orders;
-pub mod claim_tags;
+// pub mod claim_tags;
-pub mod countries;
+// pub mod countries;
-pub mod currencies;
+// pub mod currencies;
-pub mod custom_shipping_options;
+// pub mod custom_shipping_options;
-pub mod customer_group_customers;
+// pub mod customer_group_customers;
-pub mod customer_groups;
+// pub mod customer_groups;
-pub mod customers;
+// pub mod customers;
-pub mod discount_condition_customer_groups;
+// pub mod discount_condition_customer_groups;
-pub mod discount_condition_product_collections;
+// pub mod discount_condition_product_collections;
-pub mod discount_condition_product_tags;
+// pub mod discount_condition_product_tags;
-pub mod discount_condition_product_types;
+// pub mod discount_condition_product_types;
-pub mod discount_condition_products;
+// pub mod discount_condition_products;
-pub mod discount_conditions;
+// pub mod discount_conditions;
-pub mod discount_regions;
+// pub mod discount_regions;
-pub mod discount_rule_products;
+// pub mod discount_rule_products;
-pub mod discount_rules;
+// pub mod discount_rules;
-pub mod discounts;
+// pub mod discounts;
-pub mod draft_orders;
+// pub mod draft_orders;
-pub mod fulfillment_items;
+// pub mod fulfillment_items;
-pub mod fulfillment_providers;
+// pub mod fulfillment_providers;
-pub mod fulfillments;
+// pub mod fulfillments;
-pub mod gift_card_transactions;
+// pub mod gift_card_transactions;
-pub mod gift_cards;
+// pub mod gift_cards;
-pub mod idempotency_keys;
+// pub mod idempotency_keys;
-pub mod images;
+// pub mod images;
-pub mod invites;
+// pub mod invites;
-pub mod line_item_adjustments;
+// pub mod line_item_adjustments;
-pub mod line_item_tax_lines;
+// pub mod line_item_tax_lines;
-pub mod line_items;
+// pub mod line_items;
-pub mod migrations;
+// pub mod migrations;
-pub mod money_amounts;
+// pub mod money_amounts;
-pub mod notes;
+// pub mod notes;
-pub mod notification_providers;
+// pub mod notification_providers;
-pub mod notifications;
+// pub mod notifications;
-pub mod oauth;
+// pub mod oauth;
-pub mod order_discounts;
+// pub mod order_discounts;
-pub mod order_edits;
+// pub mod order_edits;
-pub mod order_gift_cards;
+// pub mod order_gift_cards;
-pub mod order_item_changes;
+// pub mod order_item_changes;
-pub mod orders;
+// pub mod orders;
-pub mod payment_collection_payments;
+// pub mod payment_collection_payments;
-pub mod payment_collection_sessions;
+// pub mod payment_collection_sessions;
-pub mod payment_collections;
+// pub mod payment_collections;
-pub mod payment_providers;
+// pub mod payment_providers;
-pub mod payment_sessions;
+// pub mod payment_sessions;
-pub mod payments;
+// pub mod payments;
-pub mod price_list_customer_groups;
+// pub mod price_list_customer_groups;
-pub mod price_lists;
+// pub mod price_lists;
-pub mod product_categories;
+// pub mod product_categories;
-pub mod product_category_products;
+// pub mod product_category_products;
-pub mod product_collections;
+// pub mod product_collections;
-pub mod product_images;
+// pub mod product_images;
-pub mod product_option_values;
+// pub mod product_option_values;
-pub mod product_options;
+// pub mod product_options;
-pub mod product_sales_channels;
+// pub mod product_sales_channels;
-pub mod product_tags;
+// pub mod product_tags;
-pub mod product_tax_rates;
+// pub mod product_tax_rates;
-pub mod product_to_tags;
+// pub mod product_to_tags;
-pub mod product_type_tax_rates;
+// pub mod product_type_tax_rates;
-pub mod product_types;
+// pub mod product_types;
-pub mod product_variant_inventory_items;
+// pub mod product_variant_inventory_items;
-pub mod product_variants;
+// pub mod product_variants;
-pub mod products;
+// pub mod products;
-pub mod publishable_api_key_sales_channels;
+// pub mod publishable_api_key_sales_channels;
-pub mod publishable_api_keys;
+// pub mod publishable_api_keys;
-pub mod refunds;
+// pub mod refunds;
-pub mod region_fulfillment_providers;
+// pub mod region_fulfillment_providers;
-pub mod region_payment_providers;
+// pub mod region_payment_providers;
-pub mod regions;
+// pub mod regions;
-pub mod return_items;
+// pub mod return_items;
-pub mod return_reasons;
+// pub mod return_reasons;
-pub mod returns;
+// pub mod returns;
-pub mod sales_channel_locations;
+// pub mod sales_channel_locations;
-pub mod sales_channels;
+// pub mod sales_channels;
-pub mod sea_orm_active_enums;
+// pub mod sea_orm_active_enums;
-pub mod shipping_method_tax_lines;
+// pub mod shipping_method_tax_lines;
-pub mod shipping_methods;
+// pub mod shipping_methods;
-pub mod shipping_option_requirements;
+// pub mod shipping_option_requirements;
-pub mod shipping_options;
+// pub mod shipping_options;
-pub mod shipping_profiles;
+// pub mod shipping_profiles;
-pub mod shipping_tax_rates;
+// pub mod shipping_tax_rates;
-pub mod staged_jobs;
+// pub mod staged_jobs;
-pub mod store_currencies;
+// pub mod store_currencies;
-pub mod stores;
+// pub mod stores;
-pub mod swaps;
+// pub mod swaps;
-pub mod tax_providers;
+// pub mod tax_providers;
-pub mod tax_rates;
+// pub mod tax_rates;
-pub mod tracking_links;
+// pub mod tracking_links;
-pub mod users;
+// pub mod users;

crates/telemetry-plugin/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "plugin-telemetry"
+name = "telemetry-plugin"
 version = "0.1.0"
 edition = "2021"
 
@@ -7,10 +7,14 @@ edition = "2021"
 crate-type = ['dylib']
 
 [dependencies]
-opentelemetry-otlp = { version = "0.12.0", features = ['trace', 'serde'] }
+opentelemetry-otlp = { version = "0.12.0", features = ['trace', 'serde', 'reqwest', 'opentelemetry-http', 'metrics', 'grpc-tonic', 'reqwest-client'] }
-opentelemetry = { version = "0.19.0", features = ['default', 'trace'] }
+opentelemetry = { version = "0.19.0", features = ['default', 'trace', 'metrics', 'rt-tokio'] }
 tokio = { version = "1.28.2", features = ['full'] }
 plugin-api = { workspace = true }
 tracing-bunyan-formatter = { version = "0.3.7" }
 tracing = { version = "0.1.37" }
 serde = { version = "1.0.164", features = ['derive'] }
+async-trait = { version = "0.1.68" }
+opentelemetry-semantic-conventions = { version = "0.11.0" }
+tracing-opentelemetry = { version = "0.19.0", features = ['async-trait', 'metrics'] }
+tracing-subscriber = { version = "0.3.17", features = ['env-filter'] }

crates/telemetry-plugin/src/lib.rs

@@ -1,10 +1,13 @@
 use opentelemetry::sdk::propagation::TraceContextPropagator;
-use opentelemetry::sdk::Resource;
+use opentelemetry::sdk::{trace, Resource};
-use opentelemetry::{global, trace, KeyValue};
+use opentelemetry::{global, KeyValue};
 use opentelemetry_otlp::WithExportConfig;
 use plugin_api::{AppConfig, PluginType};
 use serde::Deserialize;
 use tracing_bunyan_formatter::{BunyanFormattingLayer, JsonStorageLayer};
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+use tracing_subscriber::{EnvFilter, Registry};
 
 pub fn init_telemetry(config: &TelemetryConfig) {
     let exporter = opentelemetry_otlp::new_exporter()
@@ -28,6 +31,7 @@ pub fn init_telemetry(config: &TelemetryConfig) {
     let level_filter_layer = EnvFilter::try_from_default_env().unwrap_or(EnvFilter::new(
         &config
             .log_level
+            .clone()
             .or_else(|| std::env::var("RUST_LOG").ok())
             .unwrap_or_else(|| "INFO".to_string()),
     ));
@@ -53,6 +57,7 @@ pub struct TelemetryConfig {
 
 pub struct TelemetryPlugin {}
 
+#[async_trait::async_trait]
 impl plugin_api::Plugin for TelemetryPlugin {
     fn plugin_type() -> PluginType
     where

crates/web-api-plugin-tester/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "web-api-plugin-tester"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+plugin-api = { path = "../plugin-api" }
+telemetry-plugin = { path = "../telemetry-plugin" }
+actix = { version = "0.13.0" }
+actix-web = { version = "4.3.1" }
+actix-rt = { version = "2.8.0" }
+actix-web-opentelemetry = { version = "0.13.0" }
+serde = { version = "1.0.164", features = ['derive'] }
+serde_json = { version = "1.0.97" }
+futures = { version = "0.3.28" }
+rand = { version = "0.8.5" }
+reqwest = { version = "0.11.18", features = ['json', 'serde_json', 'stream', 'rustls', 'tokio-util', 'socks', 'futures-channel'] }
+toml = { version = "0.7.4" }
+tracing = { version = "0.1.37" }
+anyhow = { version = "1.0.71" }

crates/web-api-plugin-tester/src/main.rs

@@ -0,0 +1,115 @@
+use actix_web::http::StatusCode;
+use actix_web::{get, App, Error, HttpResponse, HttpServer};
+use plugin_api::{AppConfig, Plugin};
+use rand::prelude::SliceRandom;
+use reqwest::Client;
+use serde::*;
+use telemetry_plugin::TelemetryPlugin;
+use tracing::instrument;
+
+#[actix_web::main]
+async fn main() -> std::io::Result<()> {
+    let mut app_config = AppConfig::default();
+    app_config.plugin_mut().insert(
+        "telemetry".to_string(),
+        toml::Value::Table({
+            let mut t = toml::Table::with_capacity(3);
+            t.insert(
+                "endpoint".to_string(),
+                toml::Value::String("http://localhost:16686".to_string()),
+            );
+            t.insert(
+                "log_level".to_string(),
+                toml::Value::String("DEBUG".to_string()),
+            );
+            t.insert(
+                "service_name".to_string(),
+                toml::Value::String("quickwit-example".to_string()),
+            );
+            t
+        }),
+    );
+
+    TelemetryPlugin::initialize(&app_config).await;
+
+    HttpServer::new(|| {
+        App::new()
+            .wrap(actix_web_opentelemetry::RequestTracing::new())
+            .service(get_posts)
+    })
+    .bind(("127.0.0.1", 8080))?
+    .run()
+    .await
+}
+
+const BASE_API_URL: &'static str = "https://jsonplaceholder.typicode.com";
+
+// The get_post handler
+#[instrument(level = "info", name = "get_posts", skip_all)]
+#[get("/posts")]
+async fn get_posts() -> Result<HttpResponse, Error> {
+    // Randomly simulate errors in request handling
+    let choices = [200, 400, 401, 200, 500, 501, 200, 500];
+    let mut rng = rand::thread_rng();
+    let choice = choices.choose(&mut rng).unwrap().clone();
+    match choice {
+        400..=401 => Ok(HttpResponse::new(StatusCode::from_u16(choice).unwrap())),
+        500..=501 => Ok(HttpResponse::new(StatusCode::from_u16(choice).unwrap())),
+        _ => {
+            let posts = fetch_posts(20)
+                .await
+                .map_err(actix_web::error::ErrorInternalServerError)?;
+            Ok(HttpResponse::Ok().json(posts))
+        }
+    }
+}
+
+// Fetching posts with a limit.
+#[instrument(level = "info", name = "fetch_posts")]
+async fn fetch_posts(limit: usize) -> anyhow::Result<Vec<Post>> {
+    let client = Client::new();
+    let url = format!("{BASE_API_URL}/posts");
+    let mut posts: Vec<Post> = client.get(url).send().await?.json().await?;
+    posts.truncate(limit);
+    let post_idx_to_ids: Vec<(usize, i64)> = posts
+        .iter()
+        .enumerate()
+        .map(|(idx, post)| (idx, post.id))
+        .collect();
+
+    // fetch post comments one after another.
+    for (index, post_id) in post_idx_to_ids {
+        let comments = fetch_comments(&client, post_id).await?;
+        posts[index].comments = comments
+    }
+
+    Ok(posts)
+}
+
+#[instrument(level = "info", name = "fetch_comments")]
+async fn fetch_comments(client: &Client, id: i64) -> anyhow::Result<Vec<Comment>> {
+    let url = format!("{BASE_API_URL}/posts/{id}/comments");
+    let mut comments: Vec<Comment> = client.get(url).send().await?.json().await?;
+    Ok(comments)
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Post {
+    pub user_id: i64,
+    pub id: i64,
+    pub title: String,
+    pub body: String,
+    #[serde(default)]
+    pub comments: Vec<Comment>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Comment {
+    pub post_id: i64,
+    pub id: i64,
+    pub name: String,
+    pub email: String,
+    pub body: String,
+}

grafana-storage/alerting/1/__default__.tmpl

@@ -0,0 +1,53 @@
+
+{{ define "__subject" }}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ if gt (.Alerts.Resolved | len) 0 }}, RESOLVED:{{ .Alerts.Resolved | len }}{{ end }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join " " }} {{ if gt (len .CommonLabels) (len .GroupLabels) }}({{ with .CommonLabels.Remove .GroupLabels.Names }}{{ .Values | join " " }}{{ end }}){{ end }}{{ end }}
+
+{{ define "__text_values_list" }}{{ if len .Values }}{{ $first := true }}{{ range $refID, $value := .Values -}}
+{{ if $first }}{{ $first = false }}{{ else }}, {{ end }}{{ $refID }}={{ $value }}{{ end -}}
+{{ else }}[no value]{{ end }}{{ end }}
+
+{{ define "__text_alert_list" }}{{ range . }}
+Value: {{ template "__text_values_list" . }}
+Labels:
+{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}Annotations:
+{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}{{ if gt (len .GeneratorURL) 0 }}Source: {{ .GeneratorURL }}
+{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: {{ .SilenceURL }}
+{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: {{ .DashboardURL }}
+{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: {{ .PanelURL }}
+{{ end }}{{ end }}{{ end }}
+
+{{ define "default.title" }}{{ template "__subject" . }}{{ end }}
+
+{{ define "default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing**
+{{ template "__text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}
+
+{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**
+{{ template "__text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }}
+
+
+{{ define "__teams_text_alert_list" }}{{ range . }}
+Value: {{ template "__text_values_list" . }}
+Labels:
+{{ range .Labels.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}
+Annotations:
+{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}
+{{ end }}
+{{ if gt (len .GeneratorURL) 0 }}Source: [{{ .GeneratorURL }}]({{ .GeneratorURL }})
+
+{{ end }}{{ if gt (len .SilenceURL) 0 }}Silence: [{{ .SilenceURL }}]({{ .SilenceURL }})
+
+{{ end }}{{ if gt (len .DashboardURL) 0 }}Dashboard: [{{ .DashboardURL }}]({{ .DashboardURL }})
+
+{{ end }}{{ if gt (len .PanelURL) 0 }}Panel: [{{ .PanelURL }}]({{ .PanelURL }})
+
+{{ end }}
+{{ end }}{{ end }}
+
+
+{{ define "teams.default.message" }}{{ if gt (len .Alerts.Firing) 0 }}**Firing**
+{{ template "__teams_text_alert_list" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}
+
+{{ end }}{{ end }}{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**
+{{ template "__teams_text_alert_list" .Alerts.Resolved }}{{ end }}{{ end }}

qwdata/queues/partition_id

@@ -0,0 +1 @@
+ingest_partition_01H3FQJ4S8FCJRDTW2KNVMPKBF