diff --git a/Cargo.lock b/Cargo.lock index ecb0f4e..21b300c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,7 +345,7 @@ dependencies = [ [[package]] name = "databricks_kube" -version = "0.3.2" +version = "0.3.3" dependencies = [ "assert-json-diff", "async-stream", diff --git a/charts/databricks-kube-operator/Chart.yaml b/charts/databricks-kube-operator/Chart.yaml index 4fe7490..9101e95 100644 --- a/charts/databricks-kube-operator/Chart.yaml +++ b/charts/databricks-kube-operator/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.2 +version: 0.3.3 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/databricks-kube-operator/templates/sts.yaml b/charts/databricks-kube-operator/templates/sts.yaml index fd9aab3..4d5423a 100644 --- a/charts/databricks-kube-operator/templates/sts.yaml +++ b/charts/databricks-kube-operator/templates/sts.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 10 containers: - name: dko - image: ghcr.io/mach-kernel/databricks-kube-operator:0.3.2 + image: ghcr.io/mach-kernel/databricks-kube-operator:0.3.3 imagePullPolicy: Always env: - name: DATABRICKS_KUBE_CONFIGMAP diff --git a/databricks-kube/Cargo.toml b/databricks-kube/Cargo.toml index 9d655ad..f98236e 100644 --- a/databricks-kube/Cargo.toml +++ b/databricks-kube/Cargo.toml @@ -5,7 +5,7 @@ path = "src/crdgen.rs" [package] name = "databricks_kube" default-run = "databricks_kube" -version = "0.3.2" +version = "0.3.3" edition = "2021" [dependencies] diff --git a/databricks-kube/src/crds/databricks_job.rs b/databricks-kube/src/crds/databricks_job.rs index 9c21a29..980974b 100644 --- a/databricks-kube/src/crds/databricks_job.rs +++ b/databricks-kube/src/crds/databricks_job.rs @@ -1,16 +1,19 @@ use std::collections::hash_map::DefaultHasher; use std::hash::Hasher; +use std::pin::Pin; use std::sync::Arc; use std::time::SystemTime; -use std::{hash::Hash, pin::Pin}; use crate::traits::remote_api_status::RemoteAPIStatus; +use crate::util::hash_json_value; use crate::{ context::Context, error::DatabricksKubeError, traits::remote_api_resource::RemoteAPIResource, traits::rest_config::RestConfig, }; -use databricks_rust_jobs::models::{JobsRunsList200Response, RunLifeCycleState, RunState, JobSettings}; +use databricks_rust_jobs::models::{ + JobSettings, JobsRunsList200Response, RunLifeCycleState, RunState, +}; use databricks_rust_jobs::{ apis::default_api, models::{ @@ -155,17 +158,14 @@ impl DatabricksJob { fn hash_run_request(request: &JobsRunNowRequest, settings: Option>) -> u64 { let mut hasher = DefaultHasher::new(); - let request_as_json = serde_json::to_string(&request).unwrap(); + let request_as_value = serde_json::to_value(&request).unwrap(); + hash_json_value(&mut hasher, &request_as_value); if let Some(settings) = settings { - let settings_as_json = serde_json::to_string(&settings).unwrap(); - settings_as_json.hash(&mut hasher); + let settings_as_value = serde_json::to_value(&settings).unwrap(); + hash_json_value(&mut hasher, &settings_as_value); } - request_as_json.hash(&mut hasher); - - // Databricks docs state a 64 char limit for the idempotency token, - // so we can get away with coercing i64 to a string hasher.finish() } } @@ -221,7 +221,8 @@ impl RemoteAPIResource for DatabricksJob { job_id, ..run_request.unwrap() }; - run_request.idempotency_token = Some(Self::hash_run_request(&run_request, job_settings).to_string()); + run_request.idempotency_token = + Some(Self::hash_run_request(&run_request, job_settings).to_string()); let triggered = default_api::jobs_run_now(&config, Some(run_request)).await?; diff --git a/databricks-kube/src/util.rs b/databricks-kube/src/util.rs index 876afee..92a7452 100644 --- a/databricks-kube/src/util.rs +++ b/databricks-kube/src/util.rs @@ -1,5 +1,10 @@ #![allow(dead_code)] +use serde_json::{json, Value}; +use std::collections::hash_map::DefaultHasher; + +use std::hash::Hash; + use std::{sync::Arc, time::Duration}; use crate::context::CONFIGMAP_NAME; @@ -23,7 +28,6 @@ use kube::{ }; use schemars::schema_for; -use serde_json::json; use tokio::time::timeout; pub async fn watch_api_secret( @@ -220,3 +224,25 @@ pub async fn ensure_crd( .flatten() .ok_or(DatabricksKubeError::CRDMissingError(name.to_string())) } + +pub fn hash_json_value(hash: &mut DefaultHasher, value: &Value) { + match value { + Value::Null => 0.hash(hash), + Value::Bool(b) => b.hash(hash), + Value::String(s) => s.hash(hash), + Value::Number(n) => n.hash(hash), + Value::Array(a) => { + for v in a { + hash_json_value(hash, v) + } + } + Value::Object(o) => { + let mut keys: Vec = o.keys().cloned().collect(); + keys.sort(); + + for k in keys { + hash_json_value(hash, o.get(&k).unwrap()) + } + } + } +} diff --git a/databricks-kube/tests/fixtures/random-1.json b/databricks-kube/tests/fixtures/random-1.json new file mode 100644 index 0000000..709181a --- /dev/null +++ b/databricks-kube/tests/fixtures/random-1.json @@ -0,0 +1,57 @@ +{ + "97": { + "p-fQ6", + true + ], + false, + ":2-K5J?e", + "RT~otYx=s!", + "Cfj\\" + ], + "": "[ kM$", + "!m o4Cn,@J": "&{.(92,9a", + "`9M", + { + "0*{sh-G}P": " SX+;z7a8", + "": 395497447, + "D';^y')oYv": { + ";&k": [ + { + "=Q;G": -2092372186.199916, + "s*l\"4:4h": true, + "~_yT": { + "\\MS": { + "l5J": "O2]`p8y7&", + "pjGTxb9": true, + "i": [ + -810946772.6136463, + { + "d+]": 121846157, + "!:y$7&": 600716609, + "`1(Xt": "hJRqI|T", + "/)Z-]-+m": -183337327, + "8F;aGr": true, + "EF7d[|": true, + "": [ + -1822845592, + "vx", + true, + false, + "3>!Hu38", + { + "d8CgO": "#<6a]E", + "RH?,r;": [ + { + "Y": true, + "~B": { + "$.>0-": false, + "": -1290486062, + "\"ubq\"n": "j,jIPK0E", + "V:j": 5175748, + "4Ejy": true, + "0|s`": true, + "#@V": false, + "?": ":&yhB_|%j", + "7A`^q": "q)/ntAM]s", + "}s*b3GdU": -1819318896.5390286, + "=GhP2JR": 2030798676.1907976, + "]2u~{#" + }, + 1571255600, + -923922915.6178265, + 63773450.47602558, + false, + "hrSL", + -1770677847, + false, + 1578686258.642067, + "2R", + -1007382368.9073112, + -375672535.41247845, + false, + "", + "l\\#wiQu]^" + ], + "f&X": "&pWs)LC", + "?o,M?7!": 495960337, + "s8}R;,3=H:": 1779841087, + "Bm": true, + "5[h": "^Wgq[w", + "P:EuV=R?G": "", + "xQ": true, + "/[B^Gs": "!'", + "": -11644409, + "nF@": true, + "()Cj9": true + }, + true, + "^fg)EvkJ9", + -579802504.6903512, + false, + "E+d?lt}oy3", + 1946435876, + true, + false, + true + ], + "p*%tu|E5": false, + "?8jV": ":nq)y[?_nb", + "S": false, + ";K\"<[": true, + "p": -2040023949.340222, + "9Uq)8N%7d": "|~", + "nX4/M#H": true, + "E~hHW|}CM": "3" + }, + false, + false, + ",%C.4X", + "]Wje4[aT5h": "\"!m$:Z|E", + "0O!": false, + "I/;X": "nSsZU", + "w[$": "1XApP.", + "1x7Caa6": -696880401.2875822 + }, + "f,": -300667370.37430906, + "": 1246930669, + "-h8": false, + "Pi": true, + "c(z-d": false, + "#=-oY@": "k0A", + "L5d]/`zSZ|": 511115326, + "se": true, + "h~Q": "", + "D": -1994727967.7592387, + "?()NQv!-": true, + ":egFt:p~;": false, + "W~R7K{Y*@8": 675031802 + }, + "a": "L", + "Oc.": "a;:c)w2Zw+", + "kS", + "+y": "'\\Z=ngD}=s", + "x7S ;": "@$", + "@Zz3~p4'qm": true, + "!GAW%b": 1545608496.316099, + "`H$R6": true, + "G,$s//": "8", + "~JR": "Cs86*DpZ_", + "": 585803064.3783085, + "KEYK": 1457600310 + }, + "ln]", + -680565019.5444212, + 796644695, + "h<]o?z", + -533044768, + true, + false, + "Xw", + true, + "", + true, + false, + "B5", + "\\IMV" + ], + "sb\"}Ld=TN": -467913280.3991065, + "I@Hac6NpCp": false, + "W7V~/": 14751963.908402443, + "Lg+7R?3'": false, + "N9S47": 981054586.8906808, + "cJ": false, + "]B&": "q(;=~", + "K5|": -830471795, + "'7S=j": true, + "$]@bg3'hb": 450891623.6370857, + "[%RbPk\"W": true, + "A": false, + "Z3[7,ML": "1]", + "u*4i": false + }, + "Eyb=_qA": "ekIG", + "vi": "0\\R(KXo@", + "86s": -1380428171, + "?d": "V", + "C@Spa.E": 1847687035.2691035, + "l|9Z7 '.": true, + "iz[uX8": 1390815473.8063319, + "qr+m_f9": 1217711053.6633391, + ")j": 1849990336.7872872, + "Xm7,I1?": "0m*U", + "Dm]O+h[V": false + }, + false, + "@", + "&", + -2067042009, + true, + "<\\7(;Nf`9", + 569188214.0741701, + "", + -503958358.1296587, + "" + ], + "yL6*rEM", + -444763165, + 1731729474, + "VsIn1s$L", + false, + false, + "W[v)_", + false, + -575031532, + "", + true, + "ul", + "Cjm@h~", + "" + ], + "XJrlPe", + false, + "Z5", + 529611673, + true, + -1467107334.4366813, + 150819160, + false, + "X6U", + false, + ":H^T1XyH", + "s4@<|&*" + ], + "~S3*iT": "eI6@ (&\"", + "B2": true, + "jae7nL|": 990040338.7138538, + "rWL": false, + "b<<": true, + "T$jFUH": "n&llu9", + "I!lW=lC5?": false, + ")p:U((Lq": 808517314, + "y>_MC+.M": -587678639.5830443, + "B8\\q6": true, + "OkK-KI": -1786340.358038187, + "e*F=2~F": 1882903291, + "J|J'Rk\"G5 ": -1476704941, + "R7aS": "" + }, + -1026573634.2562823, + -1985126775.5408962, + false, + true, + -1487990923.407627, + false, + "BT'sK-#2t!", + 2081179044.7965076 + ], + "iH4=Wf", + true, + "eQ\\b", + 1965718287, + false, + "\\", + false, + -523909949, + 1382427823, + "(l9//%C ", + false, + "#5GynPx#", + "" + ], + true, + 1019987877.7455645, + true, + "%O", + -607043998, + -2080234834.3099074, + false, + "?6v", + false, + "LFQ", + 1987858777, + 2007501611, + "(SuKg/" + ], + ">}9T|OR4gZ", + -2133303908.2901564, + "Mh0", + "^:&1wbA5c", + true, + false, + "\"" + ], + "IXjr? I}gE": -554554692.2161977, + "2;'l{": false, + "Z": "D$": "z", + "\\(b^;iR>": 1715692319.401204, + "EsaP9LnP$": 260977819, + "D6_l@+": false, + "Q+": true, + "_66hH<#e[": "j/ervU(am", + "+&n?=5VJX?": "cG363L}", + "": -2129785679, + "i": "Zpn1\"\"T4": "rTuHc1", + "j": false, + "CC": "*4", + "HY@R": true, + "9r:\"`": 719583694, + "3v": 711584824, + "": -756360911, + "Q[1!Fk@]?zo*q": "", + "*NA": 1152785609, + "K.": "JRqb;{" + }, + -44406478, + false, + -202234593, + "=Tp.B?", + "aKr&jU;XMI", + 769835253, + -783635558, + 1079059570.6214573, + true, + "E$", + -840419363.973105, + -1507657102.6454618, + 1328188206, + true + ], + true, + 1353839590, + true, + "ag", + "gRo", + ";yZp", + false, + ";6L~EJ", + false, + "r^81O%", + "7beK|;", + true, + "Qu" + ], + "3?tB>)@p^6", + ")J\\T\\", + true, + 390943672.11548805, + -462577964.0026283, + -15115705.951853037, + 2040243993.4388747, + "", + "-b L4fA", + true, + -790822849, + "u?TnH2", + "", + "" + ], + 1300247873.5263824, + true, + 902603708, + false, + false, + "l'NZbP(", + -701793363.3665335, + 962398531.8064668, + true, + "p7^!mZmu2S" + ] \ No newline at end of file diff --git a/databricks-kube/tests/util_test.rs b/databricks-kube/tests/util_test.rs new file mode 100644 index 0000000..94e010d --- /dev/null +++ b/databricks-kube/tests/util_test.rs @@ -0,0 +1,26 @@ +use serde_json::Value; +use std::collections::hash_map::DefaultHasher; +use std::hash::Hasher; + +use databricks_kube::util::hash_json_value; + +#[test] +fn test_hash() { + let mut hasher_a = DefaultHasher::new(); + let mut hasher_b = DefaultHasher::new(); + + let fuzzy_json: Vec = vec![ + serde_json::from_str(include_str!("fixtures/random-1.json")).unwrap(), + serde_json::from_str(include_str!("fixtures/random-2.json")).unwrap(), + ]; + + for j in fuzzy_json { + hash_json_value(&mut hasher_a, &j); + hash_json_value(&mut hasher_b, &j); + + assert_eq!(hasher_a.finish(), hasher_b.finish()); + + hasher_a = DefaultHasher::new(); + hasher_b = DefaultHasher::new(); + } +}