From 9cc293a0af8e8de031281278dda945a711529fe0 Mon Sep 17 00:00:00 2001 From: Felipi Lima Matozinho Date: Tue, 30 Jul 2024 20:20:23 -0300 Subject: [PATCH 1/3] feat(examples/basic.mts): refactor code to improve readability and maintainability by adding line breaks to long statements and improving code structure feat(index.d.ts): add new classes ClusterDataSimplified, KeyspaceSimplified, TableSimplified, MaterializedViewSimplified to provide simplified cluster and keyspace information feat(index.js): update exports to include new classes ClusterDataSimplified, KeyspaceSimplified, TableSimplified, MaterializedViewSimplified for better module organization and clarity feat(scylla_session.rs): add support for simplified data structures to improve readability and maintainability of the codebase. Signed-off-by: Felipi Lima Matozinho --- examples/basic.mts | 61 +++++++--- index.d.ts | 20 +++- index.js | 6 +- src/session/scylla_session.rs | 209 +++++++++++++++++++++++++++++++++- 4 files changed, 278 insertions(+), 18 deletions(-) diff --git a/examples/basic.mts b/examples/basic.mts index 5fff6ba..240f9f3 100644 --- a/examples/basic.mts +++ b/examples/basic.mts @@ -1,4 +1,4 @@ -import { Cluster, } from "../index.js" +import { Cluster } from "../index.js"; const nodes = process.env.CLUSTER_NODES?.split(",") ?? ["127.0.0.1:9042"]; @@ -7,31 +7,62 @@ console.log(`Connecting to ${nodes}`); const cluster = new Cluster({ nodes }); const session = await cluster.connect(); -await session.execute("CREATE KEYSPACE IF NOT EXISTS basic WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"); +await session.execute( + "CREATE KEYSPACE IF NOT EXISTS basic WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }" +); await session.useKeyspace("basic"); -await session.execute("CREATE TABLE IF NOT EXISTS basic (a int, b int, c text, primary key (a, b))"); +await session.execute( + "CREATE TABLE IF NOT EXISTS basic (a int, b int, c text, primary key (a, b))" +); await session.execute("INSERT INTO basic (a, b, c) VALUES (1, 2, 'abc')"); -await session.execute("INSERT INTO basic (a, b, c) VALUES (?, ?, ?)", [3, 4, "def"]); +await session.execute("INSERT INTO basic (a, b, c) VALUES (?, ?, ?)", [ + 3, + 4, + "def", +]); -const prepared = await session.prepare("INSERT INTO basic (a, b, c) VALUES (?, 7, ?)"); +const prepared = await session.prepare( + "INSERT INTO basic (a, b, c) VALUES (?, 7, ?)" +); await session.execute(prepared, [42, "I'm prepared!"]); await session.execute(prepared, [43, "I'm prepared 2!"]); await session.execute(prepared, [44, "I'm prepared 3!"]); +const clusterData = await session.getClusterData(); +const keyspaceInfo = clusterData.getKeyspaceInfo(); + +if (keyspaceInfo) { + console.debug("========================================================"); + Object.entries(keyspaceInfo).forEach(([keyspaceName, keyspaceData]) => { + console.log(`Keyspace: ${keyspaceName}`); + // console.debug(keyspaceData.tables); + Object.entries(keyspaceData.tables).forEach(([tableName, tableData]) => { + console.log(`Table: ${tableName}:`); + console.debug("=======================") + console.debug(`partitionKey: ${tableData.partitionKey}`); + tableData?.columns?.forEach((column) => { + console.log(`Column: ${column}`); + }); + console.debug("=======================") + }); + }); + console.debug("========================================================"); +} + interface RowData { a: number; b: number; c: string; } -const result = await session.execute("SELECT a, b, c FROM basic"); -console.log(result); - -const metrics = session.metrics(); -console.log(`Queries requested: ${metrics.getQueriesNum()}`); -console.log(`Iter queries requested: ${metrics.getQueriesIterNum()}`); -console.log(`Errors occurred: ${metrics.getErrorsNum()}`); -console.log(`Iter errors occurred: ${metrics.getErrorsIterNum()}`); -console.log(`Average latency: ${metrics.getLatencyAvgMs()}`); -console.log(`99.9 latency percentile: ${metrics.getLatencyPercentileMs(99.9)}`); +// const result = await session.execute("SELECT a, b, c FROM basic"); +// console.log(result); + +// const metrics = session.metrics(); +// console.log(`Queries requested: ${metrics.getQueriesNum()}`); +// console.log(`Iter queries requested: ${metrics.getQueriesIterNum()}`); +// console.log(`Errors occurred: ${metrics.getErrorsNum()}`); +// console.log(`Iter errors occurred: ${metrics.getErrorsIterNum()}`); +// console.log(`Average latency: ${metrics.getLatencyAvgMs()}`); +// console.log(`99.9 latency percentile: ${metrics.getLatencyPercentileMs(99.9)}`); diff --git a/index.d.ts b/index.d.ts index a05e667..3584bca 100644 --- a/index.d.ts +++ b/index.d.ts @@ -121,7 +121,8 @@ export class Metrics { } export class ScyllaSession { metrics(): Metrics - execute(query: string | Query | PreparedStatement, parameters?: Array | undefined | null): Promise + getClusterData(): Promise + execute(query: string | Query | PreparedStatement, parameters?: Array | undefined | null): Promise query(scyllaQuery: Query, parameters?: Array | undefined | null): Promise prepare(query: string): Promise batch(batch: BatchStatement, parameters: Array | undefined | null>): Promise @@ -194,6 +195,23 @@ export class ScyllaSession { awaitSchemaAgreement(): Promise checkSchemaAgreement(): Promise } +export class ClusterDataSimplified { + getKeyspaceInfo(): Record | null +} +export class KeyspaceSimplified { + tables: Record + views: Record +} +export class TableSimplified { + columns: Array + partitionKey: Array + clusteringKey: Array + partitioner?: string +} +export class MaterializedViewSimplified { + viewMetadata: TableSimplified + baseTableName: string +} export class Uuid { /** Generates a random UUID v4. */ static randomV4(): Uuid diff --git a/index.js b/index.js index 2cab39a..6a039c7 100644 --- a/index.js +++ b/index.js @@ -310,7 +310,7 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { Compression, Consistency, SerialConsistency, Cluster, VerifyMode, BatchStatement, PreparedStatement, Query, Metrics, ScyllaSession, Uuid } = nativeBinding +const { Compression, Consistency, SerialConsistency, Cluster, VerifyMode, BatchStatement, PreparedStatement, Query, Metrics, ScyllaSession, ClusterDataSimplified, KeyspaceSimplified, TableSimplified, MaterializedViewSimplified, Uuid } = nativeBinding module.exports.Compression = Compression module.exports.Consistency = Consistency @@ -322,6 +322,10 @@ module.exports.PreparedStatement = PreparedStatement module.exports.Query = Query module.exports.Metrics = Metrics module.exports.ScyllaSession = ScyllaSession +module.exports.ClusterDataSimplified = ClusterDataSimplified +module.exports.KeyspaceSimplified = KeyspaceSimplified +module.exports.TableSimplified = TableSimplified +module.exports.MaterializedViewSimplified = MaterializedViewSimplified module.exports.Uuid = Uuid const customInspectSymbol = Symbol.for('nodejs.util.inspect.custom') diff --git a/src/session/scylla_session.rs b/src/session/scylla_session.rs index 7e3c3c2..22c4550 100644 --- a/src/session/scylla_session.rs +++ b/src/session/scylla_session.rs @@ -1,10 +1,15 @@ +use std::collections::HashMap; +use std::sync::Arc; + use crate::helpers::query_parameter::QueryParameter; use crate::helpers::query_results::QueryResult; use crate::query::batch_statement::ScyllaBatchStatement; use crate::query::scylla_prepared_statement::PreparedStatement; use crate::query::scylla_query::Query; use crate::types::uuid::Uuid; -use napi::bindgen_prelude::Either3; +use napi::bindgen_prelude::{Either3, FromNapiValue}; +use scylla::transport::topology::{Column, Keyspace, MaterializedView, Table, UserDefinedType}; +use scylla::transport::ClusterData; use super::metrics; @@ -13,6 +18,195 @@ pub struct ScyllaSession { session: scylla::Session, } +#[napi] +pub struct ClusterDataSimplified { + inner: Arc, +} + +impl From> for ClusterDataSimplified { + fn from(cluster_data: Arc) -> Self { + ClusterDataSimplified { + inner: cluster_data, + } + } +} + +#[napi] +impl ClusterDataSimplified { + #[napi] + pub fn get_keyspace_info(&self) -> Option> { + let keyspaces_info = self.inner.get_keyspace_info(); + + if keyspaces_info.is_empty() { + None + } else { + Some( + keyspaces_info + .into_iter() + .map(|(k, v)| (k.clone(), KeyspaceSimplified::from((*v).clone()))) + .collect(), + ) + } + } +} + +impl From for KeyspaceSimplified { + fn from(keyspace: Keyspace) -> Self { + // filter to have only the table basic + let mut keyspace_tables = HashMap::new(); + + for (table_name, table_info) in keyspace.tables.iter() { + if table_name != "basic" { + continue; + } + keyspace_tables.insert(table_name.clone(), table_info.clone()); + } + + keyspace_tables.iter().for_each(|(table_name, table_info)| { + println!(" Table: {}", table_name); + // table_info + // .columns + // .iter() + // .for_each(|(column_name, column_info)| { + // println!(" Column: {}", column_name); + // println!(" Type: {:?}", column_info); + // }); + }); + + KeyspaceSimplified { + // strategy: format!("{:?}", keyspace.strategy), + tables: keyspace_tables + .into_iter() + .map(|(k, v)| (k, TableSimplified::from(v))) + .collect(), + views: keyspace + .views + .into_iter() + .map(|(k, v)| (k, MaterializedViewSimplified::from(v))) + .collect(), + // user_defined_types: keyspace.user_defined_types.into_iter().map(|(k, v)| (k, UserDefinedTypeSimplified::from(v))).collect(), + } + } +} + +#[napi] +#[derive(Clone)] +pub struct KeyspaceSimplified { + // pub strategy: String, + pub tables: HashMap, + pub views: HashMap, + // pub user_defined_types: HashMap, +} + +impl FromNapiValue for MaterializedViewSimplified { + unsafe fn from_napi_value( + env: napi::sys::napi_env, + napi_val: napi::sys::napi_value, + ) -> napi::Result { + panic!("Not implemented") + } +} + +// impl From for KeyspaceSimplified { +// fn from(keyspace: Keyspace) -> Self { +// KeyspaceSimplified { +// // strategy: format!("{:?}", keyspace.strategy), +// tables: keyspace +// .tables +// .into_iter() +// .map(|(k, v)| (k, TableSimplified::from(v))) +// .collect(), +// views: keyspace +// .views +// .into_iter() +// .map(|(k, v)| (k, ViewSimplified::from(v))) +// .collect(), +// // user_defined_types: keyspace.user_defined_types.into_iter().map(|(k, v)| (k, UserDefinedTypeSimplified::from(v))).collect(), +// } +// } +// } + +#[napi] +#[derive(Clone)] +pub struct TableSimplified { + pub columns: Vec, + pub partition_key: Vec, + pub clustering_key: Vec, + pub partitioner: Option, +} + +impl FromNapiValue for TableSimplified { + unsafe fn from_napi_value( + env: napi::sys::napi_env, + napi_val: napi::sys::napi_value, + ) -> napi::Result { + panic!("Not implemented") + } +} + +impl From for TableSimplified { + fn from(table: Table) -> Self { + println!("Table: {:?}", table); + println!("Columns: {:?}", table.columns.clone().into_iter().map(|(k, _)| k.clone()).collect::>()); + + TableSimplified { + columns: table.columns.clone().into_iter().map(|(k, _)| (k.clone())).collect::>(), + partition_key: table.partition_key.clone(), + clustering_key: table.clustering_key.clone(), + partitioner: table.partitioner.clone(), + } + } +} + +// #[napi] +// #[derive(Clone)] +// pub struct ColumnSimplified { +// pub type_: String, +// pub kind: String, +// } + +// impl From for ColumnSimplified { +// fn from(column: Column) -> Self { +// ColumnSimplified { +// type_: format!("{:?}", column.type_), +// kind: format!("{:?}", column.kind), +// } +// } +// } + +#[napi] +#[derive(Clone)] +pub struct MaterializedViewSimplified { + pub view_metadata: TableSimplified, + pub base_table_name: String, +} + +impl From for MaterializedViewSimplified { + fn from(view: MaterializedView) -> Self { + MaterializedViewSimplified { + view_metadata: TableSimplified::from(view.view_metadata), + base_table_name: view.base_table_name, + } + } +} + +// #[napi] +// pub struct UserDefinedTypeSimplified { +// pub name: String, +// pub keyspace: String, +// // pub field_types: Vec<(String, String)>, // Simplified for the example +// } + +// impl From> for UserDefinedTypeSimplified { +// fn from(udt: Arc) -> Self { +// UserDefinedTypeSimplified { +// name: udt.name.clone(), +// keyspace: udt.keyspace.clone(), +// // field_types: udt.field_types.iter().map(|(k, v)| (k.clone(), format!("{:?}", v))).collect(), +// } +// } +// } + #[napi] impl ScyllaSession { pub fn new(session: scylla::Session) -> Self { @@ -24,6 +218,19 @@ impl ScyllaSession { metrics::Metrics::new(self.session.get_metrics()) } + #[napi] + pub async fn get_cluster_data(&self) -> ClusterDataSimplified { + self + .session + .refresh_metadata() + .await + .expect("Failed to refresh metadata"); + + let cluster_data: Arc = self.session.get_cluster_data(); + // ClusterDataSimplified::from(cluster_data) + cluster_data.into() + } + #[napi] pub async fn execute( &self, From ab4f64ad600142b15aa4b5d6c48afec0c8faa1fc Mon Sep 17 00:00:00 2001 From: Felipi Lima Matozinho Date: Wed, 31 Jul 2024 12:24:59 -0300 Subject: [PATCH 2/3] fix(basic.mts): remove unnecessary line breaks in SQL queries for creating keyspace and table feat(basic.mts): add support for fetching and displaying keyspace and table information feat(fetch-schema.mts): add new file to fetch and display keyspace and table information refactor(scylla_session.rs): restructure code to improve readability and maintainability by renaming structs and methods for better semantics and clarity. Update struct names to match their purpose and improve consistency in naming conventions. Signed-off-by: Felipi Lima Matozinho --- examples/basic.mts | 61 +++------- examples/fetch-schema.mts | 61 ++++++++++ index.d.ts | 49 +++++--- index.js | 7 +- src/session/scylla_session.rs | 219 +++++++++++++++------------------- 5 files changed, 205 insertions(+), 192 deletions(-) create mode 100644 examples/fetch-schema.mts diff --git a/examples/basic.mts b/examples/basic.mts index 240f9f3..9e47e2f 100644 --- a/examples/basic.mts +++ b/examples/basic.mts @@ -1,4 +1,4 @@ -import { Cluster } from "../index.js"; +import { Cluster } from "../index.js" const nodes = process.env.CLUSTER_NODES?.split(",") ?? ["127.0.0.1:9042"]; @@ -7,62 +7,31 @@ console.log(`Connecting to ${nodes}`); const cluster = new Cluster({ nodes }); const session = await cluster.connect(); -await session.execute( - "CREATE KEYSPACE IF NOT EXISTS basic WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }" -); +await session.execute("CREATE KEYSPACE IF NOT EXISTS basic WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }"); await session.useKeyspace("basic"); -await session.execute( - "CREATE TABLE IF NOT EXISTS basic (a int, b int, c text, primary key (a, b))" -); +await session.execute("CREATE TABLE IF NOT EXISTS basic (a int, b int, c text, primary key (a, b))"); await session.execute("INSERT INTO basic (a, b, c) VALUES (1, 2, 'abc')"); -await session.execute("INSERT INTO basic (a, b, c) VALUES (?, ?, ?)", [ - 3, - 4, - "def", -]); +await session.execute("INSERT INTO basic (a, b, c) VALUES (?, ?, ?)", [3, 4, "def"]); -const prepared = await session.prepare( - "INSERT INTO basic (a, b, c) VALUES (?, 7, ?)" -); +const prepared = await session.prepare("INSERT INTO basic (a, b, c) VALUES (?, 7, ?)"); await session.execute(prepared, [42, "I'm prepared!"]); await session.execute(prepared, [43, "I'm prepared 2!"]); await session.execute(prepared, [44, "I'm prepared 3!"]); -const clusterData = await session.getClusterData(); -const keyspaceInfo = clusterData.getKeyspaceInfo(); - -if (keyspaceInfo) { - console.debug("========================================================"); - Object.entries(keyspaceInfo).forEach(([keyspaceName, keyspaceData]) => { - console.log(`Keyspace: ${keyspaceName}`); - // console.debug(keyspaceData.tables); - Object.entries(keyspaceData.tables).forEach(([tableName, tableData]) => { - console.log(`Table: ${tableName}:`); - console.debug("=======================") - console.debug(`partitionKey: ${tableData.partitionKey}`); - tableData?.columns?.forEach((column) => { - console.log(`Column: ${column}`); - }); - console.debug("=======================") - }); - }); - console.debug("========================================================"); -} - interface RowData { a: number; b: number; c: string; } -// const result = await session.execute("SELECT a, b, c FROM basic"); -// console.log(result); - -// const metrics = session.metrics(); -// console.log(`Queries requested: ${metrics.getQueriesNum()}`); -// console.log(`Iter queries requested: ${metrics.getQueriesIterNum()}`); -// console.log(`Errors occurred: ${metrics.getErrorsNum()}`); -// console.log(`Iter errors occurred: ${metrics.getErrorsIterNum()}`); -// console.log(`Average latency: ${metrics.getLatencyAvgMs()}`); -// console.log(`99.9 latency percentile: ${metrics.getLatencyPercentileMs(99.9)}`); +const result = await session.execute("SELECT a, b, c FROM basic"); +console.log(result); + +const metrics = session.metrics(); +console.log(`Queries requested: ${metrics.getQueriesNum()}`); +console.log(`Iter queries requested: ${metrics.getQueriesIterNum()}`); +console.log(`Errors occurred: ${metrics.getErrorsNum()}`); +console.log(`Iter errors occurred: ${metrics.getErrorsIterNum()}`); +console.log(`Average latency: ${metrics.getLatencyAvgMs()}`); +console.log(`99.9 latency percentile: ${metrics.getLatencyPercentileMs(99.9)}`); \ No newline at end of file diff --git a/examples/fetch-schema.mts b/examples/fetch-schema.mts new file mode 100644 index 0000000..65f7868 --- /dev/null +++ b/examples/fetch-schema.mts @@ -0,0 +1,61 @@ +import { Cluster } from "../index.js"; + +const nodes = process.env.CLUSTER_NODES?.split(",") ?? ["127.0.0.1:9042"]; + +console.log(`Connecting to ${nodes}`); + +const cluster = new Cluster({ nodes }); +const session = await cluster.connect(); + +await session.execute( + "CREATE KEYSPACE IF NOT EXISTS basic WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }", +); +await session.useKeyspace("basic"); + +await session.execute( + "CREATE TABLE IF NOT EXISTS basic (a int, b int, c text, primary key (a, b))", +); + +await session.execute("INSERT INTO basic (a, b, c) VALUES (1, 2, 'abc')"); +await session.execute("INSERT INTO basic (a, b, c) VALUES (?, ?, ?)", [ + 3, + 4, + "def", +]); + +const prepared = await session.prepare( + "INSERT INTO basic (a, b, c) VALUES (?, 7, ?)", +); +await session.execute(prepared, [42, "I'm prepared!"]); +await session.execute(prepared, [43, "I'm prepared 2!"]); +await session.execute(prepared, [44, "I'm prepared 3!"]); + +const clusterData = await session.getClusterData(); +const keyspaceInfo = clusterData.getKeyspaceInfo(); + +if (keyspaceInfo) { + // biome-ignore lint/complexity/noForEach: + Object.entries(keyspaceInfo).forEach(([keyspaceName, keyspaceData]) => { + console.debug("========================================================"); + console.log(`Keyspace: ${keyspaceName}`); + console.debug( + `replication strategy: ${keyspaceData.strategy.kind}:`, + keyspaceData.strategy.data, + ); + // biome-ignore lint/complexity/noForEach: + Object.entries(keyspaceData.tables).forEach(([tableName, tableData]) => { + console.debug("-----------------------"); + console.log(`Table: ${tableName}`); + console.debug(`partitionKey: ${tableData.partitionKey}`); + console.debug(`clusteringKey: ${tableData.clusteringKey}`); + console.debug("columns: "); + // biome-ignore lint/complexity/noForEach: + tableData?.columns?.forEach((column) => { + console.log(` Column: ${column}`); + }); + console.debug("-----------------------"); + }); + console.debug("========================================================"); + }); +} + diff --git a/index.d.ts b/index.d.ts index 3584bca..48c3f9a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -59,6 +59,35 @@ export const enum VerifyMode { None = 0, Peer = 1 } +export interface SimpleStrategy { + replicationFactor: number +} +export interface NetworkTopologyStrategy { + datacenterRepfactors: Record +} +export interface Other { + name: string + data: Record +} +export interface ScyllaStrategy { + kind: string + data?: SimpleStrategy | NetworkTopologyStrategy | Other +} +export interface ScyllaKeyspace { + strategy: ScyllaStrategy + tables: Record + views: Record +} +export interface ScyllaTable { + columns: Array + partitionKey: Array + clusteringKey: Array + partitioner?: string +} +export interface ScyllaMaterializedView { + viewMetadata: ScyllaTable + baseTableName: string +} export type ScyllaCluster = Cluster export class Cluster { /** @@ -121,7 +150,7 @@ export class Metrics { } export class ScyllaSession { metrics(): Metrics - getClusterData(): Promise + getClusterData(): Promise execute(query: string | Query | PreparedStatement, parameters?: Array | undefined | null): Promise query(scyllaQuery: Query, parameters?: Array | undefined | null): Promise prepare(query: string): Promise @@ -195,22 +224,8 @@ export class ScyllaSession { awaitSchemaAgreement(): Promise checkSchemaAgreement(): Promise } -export class ClusterDataSimplified { - getKeyspaceInfo(): Record | null -} -export class KeyspaceSimplified { - tables: Record - views: Record -} -export class TableSimplified { - columns: Array - partitionKey: Array - clusteringKey: Array - partitioner?: string -} -export class MaterializedViewSimplified { - viewMetadata: TableSimplified - baseTableName: string +export class ScyllaClusterData { + getKeyspaceInfo(): Record | null } export class Uuid { /** Generates a random UUID v4. */ diff --git a/index.js b/index.js index 6a039c7..38f933a 100644 --- a/index.js +++ b/index.js @@ -310,7 +310,7 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { Compression, Consistency, SerialConsistency, Cluster, VerifyMode, BatchStatement, PreparedStatement, Query, Metrics, ScyllaSession, ClusterDataSimplified, KeyspaceSimplified, TableSimplified, MaterializedViewSimplified, Uuid } = nativeBinding +const { Compression, Consistency, SerialConsistency, Cluster, VerifyMode, BatchStatement, PreparedStatement, Query, Metrics, ScyllaSession, ScyllaClusterData, Uuid } = nativeBinding module.exports.Compression = Compression module.exports.Consistency = Consistency @@ -322,10 +322,7 @@ module.exports.PreparedStatement = PreparedStatement module.exports.Query = Query module.exports.Metrics = Metrics module.exports.ScyllaSession = ScyllaSession -module.exports.ClusterDataSimplified = ClusterDataSimplified -module.exports.KeyspaceSimplified = KeyspaceSimplified -module.exports.TableSimplified = TableSimplified -module.exports.MaterializedViewSimplified = MaterializedViewSimplified +module.exports.ScyllaClusterData = ScyllaClusterData module.exports.Uuid = Uuid const customInspectSymbol = Symbol.for('nodejs.util.inspect.custom') diff --git a/src/session/scylla_session.rs b/src/session/scylla_session.rs index 22c4550..b431aae 100644 --- a/src/session/scylla_session.rs +++ b/src/session/scylla_session.rs @@ -7,8 +7,8 @@ use crate::query::batch_statement::ScyllaBatchStatement; use crate::query::scylla_prepared_statement::PreparedStatement; use crate::query::scylla_query::Query; use crate::types::uuid::Uuid; -use napi::bindgen_prelude::{Either3, FromNapiValue}; -use scylla::transport::topology::{Column, Keyspace, MaterializedView, Table, UserDefinedType}; +use napi::bindgen_prelude::Either3; +use scylla::transport::topology::{Keyspace, MaterializedView, Strategy, Table}; use scylla::transport::ClusterData; use super::metrics; @@ -19,22 +19,22 @@ pub struct ScyllaSession { } #[napi] -pub struct ClusterDataSimplified { +pub struct ScyllaClusterData { inner: Arc, } -impl From> for ClusterDataSimplified { +impl From> for ScyllaClusterData { fn from(cluster_data: Arc) -> Self { - ClusterDataSimplified { + ScyllaClusterData { inner: cluster_data, } } } #[napi] -impl ClusterDataSimplified { +impl ScyllaClusterData { #[napi] - pub fn get_keyspace_info(&self) -> Option> { + pub fn get_keyspace_info(&self) -> Option> { let keyspaces_info = self.inner.get_keyspace_info(); if keyspaces_info.is_empty() { @@ -43,114 +43,119 @@ impl ClusterDataSimplified { Some( keyspaces_info .into_iter() - .map(|(k, v)| (k.clone(), KeyspaceSimplified::from((*v).clone()))) + .map(|(k, v)| (k.clone(), ScyllaKeyspace::from((*v).clone()))) .collect(), ) } } } -impl From for KeyspaceSimplified { +impl From for ScyllaKeyspace { fn from(keyspace: Keyspace) -> Self { - // filter to have only the table basic - let mut keyspace_tables = HashMap::new(); - - for (table_name, table_info) in keyspace.tables.iter() { - if table_name != "basic" { - continue; - } - keyspace_tables.insert(table_name.clone(), table_info.clone()); - } - - keyspace_tables.iter().for_each(|(table_name, table_info)| { - println!(" Table: {}", table_name); - // table_info - // .columns - // .iter() - // .for_each(|(column_name, column_info)| { - // println!(" Column: {}", column_name); - // println!(" Type: {:?}", column_info); - // }); - }); - - KeyspaceSimplified { - // strategy: format!("{:?}", keyspace.strategy), - tables: keyspace_tables + ScyllaKeyspace { + tables: keyspace + .tables .into_iter() - .map(|(k, v)| (k, TableSimplified::from(v))) + .map(|(k, v)| (k, ScyllaTable::from(v))) .collect(), views: keyspace .views .into_iter() - .map(|(k, v)| (k, MaterializedViewSimplified::from(v))) + .map(|(k, v)| (k, ScyllaMaterializedView::from(v))) .collect(), + strategy: keyspace.strategy.into(), // user_defined_types: keyspace.user_defined_types.into_iter().map(|(k, v)| (k, UserDefinedTypeSimplified::from(v))).collect(), } } } +#[napi(object)] +#[derive(Clone)] +pub struct SimpleStrategy { + pub replication_factor: u32, +} -#[napi] +#[napi(object)] #[derive(Clone)] -pub struct KeyspaceSimplified { - // pub strategy: String, - pub tables: HashMap, - pub views: HashMap, - // pub user_defined_types: HashMap, +pub struct NetworkTopologyStrategy { + pub datacenter_repfactors: HashMap, +} + +#[napi(object)] +#[derive(Clone)] +pub struct Other { + pub name: String, + pub data: HashMap, +} + +#[napi(object)] +#[derive(Clone)] +pub struct ScyllaStrategy { + pub kind: String, + pub data: Option>, } -impl FromNapiValue for MaterializedViewSimplified { - unsafe fn from_napi_value( - env: napi::sys::napi_env, - napi_val: napi::sys::napi_value, - ) -> napi::Result { - panic!("Not implemented") +impl From for ScyllaStrategy { + fn from(strategy: Strategy) -> Self { + match strategy { + Strategy::SimpleStrategy { replication_factor } => ScyllaStrategy { + kind: "SimpleStrategy".to_string(), + data: Some(Either3::A(SimpleStrategy { + replication_factor: replication_factor as u32, + })), + }, + Strategy::NetworkTopologyStrategy { + datacenter_repfactors, + } => ScyllaStrategy { + kind: "NetworkTopologyStrategy".to_string(), + data: Some(Either3::B(NetworkTopologyStrategy { + datacenter_repfactors: datacenter_repfactors + .into_iter() + .map(|(k, v)| (k, v as i32)) + .collect(), + })), + }, + Strategy::Other { name, data } => ScyllaStrategy { + kind: name.clone(), + data: Some(Either3::C(Other { + name: name.clone(), + data, + })), + }, + Strategy::LocalStrategy => ScyllaStrategy { + kind: "LocalStrategy".to_string(), + data: None, + }, + } } } -// impl From for KeyspaceSimplified { -// fn from(keyspace: Keyspace) -> Self { -// KeyspaceSimplified { -// // strategy: format!("{:?}", keyspace.strategy), -// tables: keyspace -// .tables -// .into_iter() -// .map(|(k, v)| (k, TableSimplified::from(v))) -// .collect(), -// views: keyspace -// .views -// .into_iter() -// .map(|(k, v)| (k, ViewSimplified::from(v))) -// .collect(), -// // user_defined_types: keyspace.user_defined_types.into_iter().map(|(k, v)| (k, UserDefinedTypeSimplified::from(v))).collect(), -// } -// } -// } +#[napi(object)] +#[derive(Clone)] +pub struct ScyllaKeyspace { + pub strategy: ScyllaStrategy, + pub tables: HashMap, + pub views: HashMap, + // pub user_defined_types: HashMap, +} -#[napi] +#[napi(object)] #[derive(Clone)] -pub struct TableSimplified { +pub struct ScyllaTable { pub columns: Vec, pub partition_key: Vec, pub clustering_key: Vec, pub partitioner: Option, } -impl FromNapiValue for TableSimplified { - unsafe fn from_napi_value( - env: napi::sys::napi_env, - napi_val: napi::sys::napi_value, - ) -> napi::Result { - panic!("Not implemented") - } -} - -impl From
for TableSimplified { +impl From
for ScyllaTable { fn from(table: Table) -> Self { - println!("Table: {:?}", table); - println!("Columns: {:?}", table.columns.clone().into_iter().map(|(k, _)| k.clone()).collect::>()); - - TableSimplified { - columns: table.columns.clone().into_iter().map(|(k, _)| (k.clone())).collect::>(), + ScyllaTable { + columns: table + .columns + .clone() + .into_iter() + .map(|(k, _)| k) + .collect::>(), partition_key: table.partition_key.clone(), clustering_key: table.clustering_key.clone(), partitioner: table.partitioner.clone(), @@ -158,55 +163,22 @@ impl From
for TableSimplified { } } -// #[napi] -// #[derive(Clone)] -// pub struct ColumnSimplified { -// pub type_: String, -// pub kind: String, -// } - -// impl From for ColumnSimplified { -// fn from(column: Column) -> Self { -// ColumnSimplified { -// type_: format!("{:?}", column.type_), -// kind: format!("{:?}", column.kind), -// } -// } -// } - -#[napi] +#[napi(object)] #[derive(Clone)] -pub struct MaterializedViewSimplified { - pub view_metadata: TableSimplified, +pub struct ScyllaMaterializedView { + pub view_metadata: ScyllaTable, pub base_table_name: String, } -impl From for MaterializedViewSimplified { +impl From for ScyllaMaterializedView { fn from(view: MaterializedView) -> Self { - MaterializedViewSimplified { - view_metadata: TableSimplified::from(view.view_metadata), + ScyllaMaterializedView { + view_metadata: ScyllaTable::from(view.view_metadata), base_table_name: view.base_table_name, } } } -// #[napi] -// pub struct UserDefinedTypeSimplified { -// pub name: String, -// pub keyspace: String, -// // pub field_types: Vec<(String, String)>, // Simplified for the example -// } - -// impl From> for UserDefinedTypeSimplified { -// fn from(udt: Arc) -> Self { -// UserDefinedTypeSimplified { -// name: udt.name.clone(), -// keyspace: udt.keyspace.clone(), -// // field_types: udt.field_types.iter().map(|(k, v)| (k.clone(), format!("{:?}", v))).collect(), -// } -// } -// } - #[napi] impl ScyllaSession { pub fn new(session: scylla::Session) -> Self { @@ -219,15 +191,14 @@ impl ScyllaSession { } #[napi] - pub async fn get_cluster_data(&self) -> ClusterDataSimplified { + pub async fn get_cluster_data(&self) -> ScyllaClusterData { self .session .refresh_metadata() .await .expect("Failed to refresh metadata"); - let cluster_data: Arc = self.session.get_cluster_data(); - // ClusterDataSimplified::from(cluster_data) + let cluster_data = self.session.get_cluster_data(); cluster_data.into() } From 6420013d2934ae13f30add81d6adf603b1a3aa69 Mon Sep 17 00:00:00 2001 From: Felipi Lima Matozinho Date: Wed, 31 Jul 2024 15:29:59 -0300 Subject: [PATCH 3/3] feat(fetch-schema.mts): refactor fetch-schema script to improve readability and error handling feat(session): add topology module to handle cluster topology information feat(session): move ScyllaClusterData struct to topology module for better organization and separation of concerns feat(topology.rs): add support for defining Scylla cluster data, keyspace, strategy, table, and materialized view structures to interact with Scylla database. Signed-off-by: Felipi Lima Matozinho --- examples/fetch-schema.mts | 86 ++++----- index.d.ts | 22 ++- src/cluster/execution_profile/consistency.rs | 1 - src/session/mod.rs | 1 + src/session/scylla_session.rs | 167 +----------------- src/session/topology.rs | 176 +++++++++++++++++++ 6 files changed, 230 insertions(+), 223 deletions(-) create mode 100644 src/session/topology.rs diff --git a/examples/fetch-schema.mts b/examples/fetch-schema.mts index 65f7868..e8bc026 100644 --- a/examples/fetch-schema.mts +++ b/examples/fetch-schema.mts @@ -7,55 +7,47 @@ console.log(`Connecting to ${nodes}`); const cluster = new Cluster({ nodes }); const session = await cluster.connect(); -await session.execute( - "CREATE KEYSPACE IF NOT EXISTS basic WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }", -); -await session.useKeyspace("basic"); - -await session.execute( - "CREATE TABLE IF NOT EXISTS basic (a int, b int, c text, primary key (a, b))", -); - -await session.execute("INSERT INTO basic (a, b, c) VALUES (1, 2, 'abc')"); -await session.execute("INSERT INTO basic (a, b, c) VALUES (?, ?, ?)", [ - 3, - 4, - "def", -]); - -const prepared = await session.prepare( - "INSERT INTO basic (a, b, c) VALUES (?, 7, ?)", -); -await session.execute(prepared, [42, "I'm prepared!"]); -await session.execute(prepared, [43, "I'm prepared 2!"]); -await session.execute(prepared, [44, "I'm prepared 3!"]); - const clusterData = await session.getClusterData(); const keyspaceInfo = clusterData.getKeyspaceInfo(); -if (keyspaceInfo) { - // biome-ignore lint/complexity/noForEach: - Object.entries(keyspaceInfo).forEach(([keyspaceName, keyspaceData]) => { - console.debug("========================================================"); - console.log(`Keyspace: ${keyspaceName}`); - console.debug( - `replication strategy: ${keyspaceData.strategy.kind}:`, - keyspaceData.strategy.data, - ); - // biome-ignore lint/complexity/noForEach: - Object.entries(keyspaceData.tables).forEach(([tableName, tableData]) => { - console.debug("-----------------------"); - console.log(`Table: ${tableName}`); - console.debug(`partitionKey: ${tableData.partitionKey}`); - console.debug(`clusteringKey: ${tableData.clusteringKey}`); - console.debug("columns: "); - // biome-ignore lint/complexity/noForEach: - tableData?.columns?.forEach((column) => { - console.log(` Column: ${column}`); - }); - console.debug("-----------------------"); - }); - console.debug("========================================================"); - }); +if (!keyspaceInfo) throw new Error("No data found"); + +console.log("ALL KEYSPACES"); +for (const keyspaceName in keyspaceInfo) { + console.log("========================================================"); + const keyspaceData = keyspaceInfo[keyspaceName]; + console.log("Keyspace: ", keyspaceName); + console.log( + "replication strategy: ", + keyspaceData.strategy.kind, + keyspaceData.strategy.data, + ); + for (const tableName in keyspaceData.tables) { + console.log("-----------------------"); + const tableData = keyspaceData.tables[tableName]; + console.log("Table: ", tableName); + console.log("partitionKey: ", tableData.partitionKey); + console.log("clusteringKey: ", tableData.clusteringKey); + console.log("columns: ", tableData.columns); + console.log("-----------------------"); + } + console.log("========================================================"); } +console.log("================== SPECIFIC KEYSPACES =================="); +console.log( + "keyspace: system_auth | strategy: ", + keyspaceInfo.system_auth.strategy, +); +console.log( + "keyspace: system_traces | strategy: ", + keyspaceInfo.system_traces.strategy, +); +console.log( + "keyspace: system_distributed_everywhere | strategy: ", + keyspaceInfo.system_distributed_everywhere.strategy, +); +console.log( + "keyspace: system_distributed | strategy: ", + keyspaceInfo.system_distributed.strategy, +); diff --git a/index.d.ts b/index.d.ts index 9d8789a..7755299 100644 --- a/index.d.ts +++ b/index.d.ts @@ -59,6 +59,15 @@ export const enum VerifyMode { None = 0, Peer = 1 } +export interface ScyllaKeyspace { + strategy: ScyllaStrategy + tables: Record + views: Record +} +export interface ScyllaStrategy { + kind: string + data?: SimpleStrategy | NetworkTopologyStrategy | Other +} export interface SimpleStrategy { replicationFactor: number } @@ -69,15 +78,6 @@ export interface Other { name: string data: Record } -export interface ScyllaStrategy { - kind: string - data?: SimpleStrategy | NetworkTopologyStrategy | Other -} -export interface ScyllaKeyspace { - strategy: ScyllaStrategy - tables: Record - views: Record -} export interface ScyllaTable { columns: Array partitionKey: Array @@ -262,6 +262,10 @@ export class ScyllaSession { checkSchemaAgreement(): Promise } export class ScyllaClusterData { + /** + * Access keyspaces details collected by the driver Driver collects various schema details like + * tables, partitioners, columns, types. They can be read using this method + */ getKeyspaceInfo(): Record | null } export class Uuid { diff --git a/src/cluster/execution_profile/consistency.rs b/src/cluster/execution_profile/consistency.rs index c75e97a..3ae8956 100644 --- a/src/cluster/execution_profile/consistency.rs +++ b/src/cluster/execution_profile/consistency.rs @@ -51,4 +51,3 @@ impl From for Consistency { } } } - diff --git a/src/session/mod.rs b/src/session/mod.rs index 023095e..2e7198a 100644 --- a/src/session/mod.rs +++ b/src/session/mod.rs @@ -1,2 +1,3 @@ pub mod metrics; pub mod scylla_session; +pub mod topology; diff --git a/src/session/scylla_session.rs b/src/session/scylla_session.rs index ac1163a..1969da2 100644 --- a/src/session/scylla_session.rs +++ b/src/session/scylla_session.rs @@ -1,6 +1,3 @@ -use std::collections::HashMap; -use std::sync::Arc; - use crate::helpers::query_parameter::QueryParameter; use crate::helpers::query_results::QueryResult; use crate::query::batch_statement::ScyllaBatchStatement; @@ -8,177 +5,15 @@ use crate::query::scylla_prepared_statement::PreparedStatement; use crate::query::scylla_query::Query; use crate::types::uuid::Uuid; use napi::bindgen_prelude::Either3; -use scylla::transport::topology::{Keyspace, MaterializedView, Strategy, Table}; -use scylla::transport::ClusterData; use super::metrics; +use super::topology::ScyllaClusterData; #[napi] pub struct ScyllaSession { session: scylla::Session, } -#[napi] -pub struct ScyllaClusterData { - inner: Arc, -} - -impl From> for ScyllaClusterData { - fn from(cluster_data: Arc) -> Self { - ScyllaClusterData { - inner: cluster_data, - } - } -} - -#[napi] -impl ScyllaClusterData { - #[napi] - pub fn get_keyspace_info(&self) -> Option> { - let keyspaces_info = self.inner.get_keyspace_info(); - - if keyspaces_info.is_empty() { - None - } else { - Some( - keyspaces_info - .into_iter() - .map(|(k, v)| (k.clone(), ScyllaKeyspace::from((*v).clone()))) - .collect(), - ) - } - } -} - -impl From for ScyllaKeyspace { - fn from(keyspace: Keyspace) -> Self { - ScyllaKeyspace { - tables: keyspace - .tables - .into_iter() - .map(|(k, v)| (k, ScyllaTable::from(v))) - .collect(), - views: keyspace - .views - .into_iter() - .map(|(k, v)| (k, ScyllaMaterializedView::from(v))) - .collect(), - strategy: keyspace.strategy.into(), - // user_defined_types: keyspace.user_defined_types.into_iter().map(|(k, v)| (k, UserDefinedTypeSimplified::from(v))).collect(), - } - } -} -#[napi(object)] -#[derive(Clone)] -pub struct SimpleStrategy { - pub replication_factor: u32, -} - -#[napi(object)] -#[derive(Clone)] -pub struct NetworkTopologyStrategy { - pub datacenter_repfactors: HashMap, -} - -#[napi(object)] -#[derive(Clone)] -pub struct Other { - pub name: String, - pub data: HashMap, -} - -#[napi(object)] -#[derive(Clone)] -pub struct ScyllaStrategy { - pub kind: String, - pub data: Option>, -} - -impl From for ScyllaStrategy { - fn from(strategy: Strategy) -> Self { - match strategy { - Strategy::SimpleStrategy { replication_factor } => ScyllaStrategy { - kind: "SimpleStrategy".to_string(), - data: Some(Either3::A(SimpleStrategy { - replication_factor: replication_factor as u32, - })), - }, - Strategy::NetworkTopologyStrategy { - datacenter_repfactors, - } => ScyllaStrategy { - kind: "NetworkTopologyStrategy".to_string(), - data: Some(Either3::B(NetworkTopologyStrategy { - datacenter_repfactors: datacenter_repfactors - .into_iter() - .map(|(k, v)| (k, v as i32)) - .collect(), - })), - }, - Strategy::Other { name, data } => ScyllaStrategy { - kind: name.clone(), - data: Some(Either3::C(Other { - name: name.clone(), - data, - })), - }, - Strategy::LocalStrategy => ScyllaStrategy { - kind: "LocalStrategy".to_string(), - data: None, - }, - } - } -} - -#[napi(object)] -#[derive(Clone)] -pub struct ScyllaKeyspace { - pub strategy: ScyllaStrategy, - pub tables: HashMap, - pub views: HashMap, - // pub user_defined_types: HashMap, -} - -#[napi(object)] -#[derive(Clone)] -pub struct ScyllaTable { - pub columns: Vec, - pub partition_key: Vec, - pub clustering_key: Vec, - pub partitioner: Option, -} - -impl From
for ScyllaTable { - fn from(table: Table) -> Self { - ScyllaTable { - columns: table - .columns - .clone() - .into_iter() - .map(|(k, _)| k) - .collect::>(), - partition_key: table.partition_key.clone(), - clustering_key: table.clustering_key.clone(), - partitioner: table.partitioner.clone(), - } - } -} - -#[napi(object)] -#[derive(Clone)] -pub struct ScyllaMaterializedView { - pub view_metadata: ScyllaTable, - pub base_table_name: String, -} - -impl From for ScyllaMaterializedView { - fn from(view: MaterializedView) -> Self { - ScyllaMaterializedView { - view_metadata: ScyllaTable::from(view.view_metadata), - base_table_name: view.base_table_name, - } - } -} - #[napi] impl ScyllaSession { pub fn new(session: scylla::Session) -> Self { diff --git a/src/session/topology.rs b/src/session/topology.rs new file mode 100644 index 0000000..8d3a878 --- /dev/null +++ b/src/session/topology.rs @@ -0,0 +1,176 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use napi::bindgen_prelude::Either3; +use scylla::transport::topology::{Keyspace, MaterializedView, Strategy, Table}; +use scylla::transport::ClusterData; + +// ============= ClusterData ============= // +#[napi] +pub struct ScyllaClusterData { + inner: Arc, +} + +impl From> for ScyllaClusterData { + fn from(cluster_data: Arc) -> Self { + ScyllaClusterData { + inner: cluster_data, + } + } +} + +#[napi] +impl ScyllaClusterData { + #[napi] + /// Access keyspaces details collected by the driver Driver collects various schema details like + /// tables, partitioners, columns, types. They can be read using this method + pub fn get_keyspace_info(&self) -> Option> { + let keyspaces_info = self.inner.get_keyspace_info(); + + if keyspaces_info.is_empty() { + None + } else { + Some( + keyspaces_info + .iter() + .map(|(k, v)| (k.clone(), ScyllaKeyspace::from((*v).clone()))) + .collect(), + ) + } + } +} +// ======================================= // + +// ============= Keyspace ============= // +#[napi(object)] +#[derive(Clone)] +pub struct ScyllaKeyspace { + pub strategy: ScyllaStrategy, + pub tables: HashMap, + pub views: HashMap, + // pub user_defined_types: HashMap, +} + +impl From for ScyllaKeyspace { + fn from(keyspace: Keyspace) -> Self { + ScyllaKeyspace { + tables: keyspace + .tables + .into_iter() + .map(|(k, v)| (k, ScyllaTable::from(v))) + .collect(), + views: keyspace + .views + .into_iter() + .map(|(k, v)| (k, ScyllaMaterializedView::from(v))) + .collect(), + strategy: keyspace.strategy.into(), + // TODO: Implement ScyllaUserDefinedType + // user_defined_types: keyspace.user_defined_types.into_iter().map(|(k, v)| (k, ScyllaUserDefinedType::from(v))).collect(), + } + } +} +// ======================================= // + +// ============= Strategy ============= // +#[napi(object)] +#[derive(Clone)] +pub struct ScyllaStrategy { + pub kind: String, + pub data: Option>, +} + +#[napi(object)] +#[derive(Clone)] +pub struct SimpleStrategy { + pub replication_factor: u32, +} + +#[napi(object)] +#[derive(Clone)] +pub struct NetworkTopologyStrategy { + pub datacenter_repfactors: HashMap, +} + +#[napi(object)] +#[derive(Clone)] +pub struct Other { + pub name: String, + pub data: HashMap, +} + +impl From for ScyllaStrategy { + fn from(strategy: Strategy) -> Self { + match strategy { + Strategy::SimpleStrategy { replication_factor } => ScyllaStrategy { + kind: "SimpleStrategy".to_string(), + data: Some(Either3::A(SimpleStrategy { + replication_factor: replication_factor as u32, + })), + }, + Strategy::NetworkTopologyStrategy { + datacenter_repfactors, + } => ScyllaStrategy { + kind: "NetworkTopologyStrategy".to_string(), + data: Some(Either3::B(NetworkTopologyStrategy { + datacenter_repfactors: datacenter_repfactors + .into_iter() + .map(|(k, v)| (k, v as i32)) + .collect(), + })), + }, + Strategy::Other { name, data } => ScyllaStrategy { + kind: name.clone(), + data: Some(Either3::C(Other { + name: name.clone(), + data, + })), + }, + Strategy::LocalStrategy => ScyllaStrategy { + kind: "LocalStrategy".to_string(), + data: None, + }, + } + } +} +// ======================================= // + +// ============= Table ============= // +#[napi(object)] +#[derive(Clone)] +pub struct ScyllaTable { + pub columns: Vec, + pub partition_key: Vec, + pub clustering_key: Vec, + pub partitioner: Option, +} + +impl From
for ScyllaTable { + fn from(table: Table) -> Self { + ScyllaTable { + columns: table.columns.clone().into_keys().collect::>(), + partition_key: table.partition_key.clone(), + clustering_key: table.clustering_key.clone(), + partitioner: table.partitioner.clone(), + } + } +} +// ======================================= // + +// ============= MaterializedView ============= // +#[napi(object)] +#[derive(Clone)] +pub struct ScyllaMaterializedView { + pub view_metadata: ScyllaTable, + pub base_table_name: String, +} + +impl From for ScyllaMaterializedView { + fn from(view: MaterializedView) -> Self { + ScyllaMaterializedView { + view_metadata: ScyllaTable::from(view.view_metadata), + base_table_name: view.base_table_name, + } + } +} +// ======================================= //