Merge pull request #62 from artefactory/dev

bump

.github/workflows/conftest.yml

@@ -0,0 +1,11 @@
+on: push
+name: Conftest
+jobs:
+  conftest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - name: test
+      uses: instrumenta/conftest-action@master
+      with:
+        files: tracking_server/Dockerfile

.gitignore

@@ -2,4 +2,12 @@
 .terraform
 *.tfstate
 *.tfstate.backup
-*.tfvars
+*.tfvars
+IaC/*/*.hcl
+IaC/*.hcl
+
+#secrets
+secrets/
+
+#python
+*/__pycache__/

IaC/main.tf

@@ -18,7 +18,7 @@
 terraform {
   backend "gcs" {
   }
-  required_version = "~> 0.13.2"
+  required_version = "> 0.13.2"
   required_providers {
     google = "~> 3.13"
   }
@@ -36,29 +36,32 @@ resource "random_id" "artifacts_bucket_name_suffix" {
   byte_length = 5
 }
 
-resource "random_password" "password" {
-  length = 16
-}
-
 module "network" {
-  source = "./modules/network"
+  source       = "./modules/network"
   network_name = var.network_name
 }
 
 module "mlflow" {
-  source = "./modules/mlflow"
-  artifacts_bucket_name = "${var.artifacts_bucket}-${random_id.artifacts_bucket_name_suffix.hex}"
-  db_password_value = random_password.password.result
-  server_docker_image = var.mlflow_docker_image
-  project_id = var.project_id
+  source                       = "./modules/mlflow"
+  mlflow_server                = var.mlflow_server
+  artifacts_bucket_name        = "${var.artifacts_bucket}-${random_id.artifacts_bucket_name_suffix.hex}"
+  server_docker_image          = var.mlflow_docker_image
+  project_id                   = var.project_id
   consent_screen_support_email = var.consent_screen_support_email
-  web_app_users = var.web_app_users
-  network_self_link = module.network.network_self_link
-  network_short_name = module.network.network_short_name
+  web_app_users                = var.web_app_users
+  network_self_link            = module.network.network_self_link
+  network_short_name           = module.network.network_short_name
+  create_default_service       = var.create_default_service == 1 ? true : false
+  oauth_client_id              = var.oauth_client_id
+  oauth_client_secret          = var.oauth_client_secret
+  create_brand                 = var.create_brand
+  brand_name                   = var.brand_name
 }
 
 module "log_pusher" {
-  source = "./modules/mlflow/log_pusher"
-  project_id = var.project_id
-  depends_on = [module.mlflow]
+  source           = "./modules/mlflow/log_pusher"
+  app_id           = module.mlflow.app_id
+  mlflow_service   = module.mlflow.mlflow_service
+  artifacts_bucket = module.mlflow.artifacts_bucket_name
+  project_id       = var.project_id
 }

IaC/modules/mlflow/artifacts/main.tf

@@ -31,4 +31,5 @@ resource "google_storage_bucket" "this" {
     }
   }
   uniform_bucket_level_access = var.storage_uniform
+  force_destroy               = true
 }

IaC/modules/mlflow/artifacts/outputs.tf

@@ -19,3 +19,8 @@ output "url" {
   description = "gcs uri"
   value       = google_storage_bucket.this.url
 }
+
+output "name" {
+  description = "gcs bucket name"
+  value       = google_storage_bucket.this.name
+}

IaC/modules/mlflow/artifacts/variables.tf

@@ -16,35 +16,35 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 variable "bucket_name" {
-    description = "Name of the bucket."
-    type        = string
+  description = "Name of the bucket."
+  type        = string
 }
 variable "bucket_location" {
-    description = "Location of the bucket."
-    type        = string
-    default     = "EUROPE-WEST1"
+  description = "Location of the bucket."
+  type        = string
+  default     = "EUROPE-WEST1"
 }
 variable "versioning_enabled" {
-    description = "True if you want to version your bucket."
-    type        = bool
-    default     = true
+  description = "True if you want to version your bucket."
+  type        = bool
+  default     = true
 }
 variable "number_of_version" {
-    description = "Number of version you want to keep with the versionning."
-    type        = number
-    default     = 1
+  description = "Number of version you want to keep with the versionning."
+  type        = number
+  default     = 1
 }
 variable "storage_class" {
-    description = "Storage class of your bucket"
-    type        = string
-    default     ="STANDARD"
+  description = "Storage class of your bucket"
+  type        = string
+  default     = "STANDARD"
 }
 variable "module_depends_on" {
   type    = any
   default = null
 }
 variable "storage_uniform" {
-    type = bool
-    description = "Wether or not uniform level acces is to be activated for the buckets"
-    default = true
+  type        = bool
+  description = "Wether or not uniform level acces is to be activated for the buckets"
+  default     = true
 }

IaC/modules/mlflow/database/main.tf

@@ -35,17 +35,18 @@ resource "google_sql_database_instance" "this_instance" {
     }
     availability_type = var.availability_type
   }
+  deletion_protection = false
 }
 
 resource "google_sql_database" "this_database" {
-  name     = var.database_name
-  instance = google_sql_database_instance.this_instance.name
+  name       = var.database_name
+  instance   = google_sql_database_instance.this_instance.name
   depends_on = [google_sql_database_instance.this_instance]
 }
 
 resource "google_sql_user" "this_user" {
-  name     = var.username
-  instance = google_sql_database_instance.this_instance.name
-  password = var.password
+  name       = var.username
+  instance   = google_sql_database_instance.this_instance.name
+  password   = var.password
   depends_on = [google_sql_database_instance.this_instance]
 }

IaC/modules/mlflow/database/variables.tf

@@ -16,42 +16,42 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 variable "instance_prefix" {
-    type = string
-    description = "Name of the database instance you want to deploy"
-    default = "mlflow"
+  type        = string
+  description = "Name of the database instance you want to deploy"
+  default     = "mlflow"
 }
 variable "database_version" {
-    type = string
-    description = "Version of the database instance you use"
-    default = "MYSQL_5_7"
+  type        = string
+  description = "Version of the database instance you use"
+  default     = "MYSQL_5_7"
 }
 variable "region" {
-    type = string
-    description = "Region of the database instance"
-    default = "europe-west1"
+  type        = string
+  description = "Region of the database instance"
+  default     = "europe-west1"
 }
 variable "size" {
-    type = string
-    description = "Size of the database instance"
-    default = "db-f1-micro"
+  type        = string
+  description = "Size of the database instance"
+  default     = "db-f1-micro"
 }
 variable "availability_type" {
-    type = string
-    description = "Availability of your instance" 
-    default = "ZONAL"
+  type        = string
+  description = "Availability of your instance"
+  default     = "ZONAL"
 }
 variable "database_name" {
-    type = string
-    description = "Name of the database created"
-    default = "mlflow"
+  type        = string
+  description = "Name of the database created"
+  default     = "mlflow"
 }
 variable "username" {
-    type = string
-    description = "Username to connect to database instance"
+  type        = string
+  description = "Username to connect to database instance"
 }
 variable "password" {
-    type = string
-    description = "Password to connect to database instance" 
+  type        = string
+  description = "Password to connect to database instance"
 }
 variable "module_depends_on" {
   type    = any

IaC/modules/mlflow/log_pusher/main.tf

@@ -20,14 +20,16 @@ resource "google_service_account" "log_pusher" {
   display_name = "mlflow log pusher"
 }
 
-resource "google_project_iam_member" "log_pusher_iap" {
+resource "google_iap_app_engine_service_iam_member" "log_pusher_iap" {
   project = var.project_id
+  app_id  = var.app_id
+  service = var.mlflow_service
   role    = "roles/iap.httpsResourceAccessor"
-  member = "serviceAccount:${google_service_account.log_pusher.email}"
+  member  = "serviceAccount:${google_service_account.log_pusher.email}"
 }
 
-resource "google_project_iam_member" "log_pusher_storage" {
-  project = var.project_id
-  role    = "roles/storage.objectAdmin"
+resource "google_storage_bucket_iam_member" "log_pusher_storage" {
+  bucket = var.artifacts_bucket
+  role   = "roles/storage.objectAdmin"
   member = "serviceAccount:${google_service_account.log_pusher.email}"
 }

IaC/modules/mlflow/log_pusher/variables.tf

@@ -17,4 +17,13 @@
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 variable "project_id" {
   type = string
+}
+variable "app_id" {
+  type = string
+}
+variable "mlflow_service" {
+  type = string
+}
+variable "artifacts_bucket" {
+  type = string
 }

IaC/modules/mlflow/main.tf

@@ -15,48 +15,53 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
 module "artifacts" {
-    source = "./artifacts"
-    bucket_name = var.artifacts_bucket_name
-    bucket_location = var.artifacts_bucket_location
-    number_of_version = var.artifacts_number_of_version
-    storage_class = var.artifacts_storage_class
+  source            = "./artifacts"
+  bucket_name       = var.artifacts_bucket_name
+  bucket_location   = var.artifacts_bucket_location
+  number_of_version = var.artifacts_number_of_version
+  storage_class     = var.artifacts_storage_class
 }
 
 module "db_secret" {
-    source = "./secret_manager"
-    secret_id = var.db_password_name
-    secret_value = var.db_password_value
+  source       = "./secret_manager"
+  secret_id    = var.db_password_name
 }
 
 module "database" {
-    source = "./database"
-    instance_prefix = var.db_instance_prefix
-    database_version = var.db_version
-    region = var.db_region
-    size = var.db_size
-    availability_type = var.db_availability_type
-    database_name = var.db_name
-    username = var.db_username
-    password = module.db_secret.secret_value
-    network_self_link = var.network_self_link
+  source            = "./database"
+  instance_prefix   = var.db_instance_prefix
+  database_version  = var.db_version
+  region            = var.db_region
+  size              = var.db_size
+  availability_type = var.db_availability_type
+  database_name     = var.db_name
+  username          = var.db_username
+  password          = module.db_secret.secret_value
+  network_self_link = var.network_self_link
 }
 
 module "server" {
-    source = "./server"
-    server_name = var.mlflow_server
-    location = var.server_location
-    docker_image_name = var.server_docker_image
-    env_variables = var.server_env_variables
-    db_private_ip = module.database.private_ip
-    project_id = var.project_id
-    db_password_name = var.db_password_name
-    db_username = var.db_username
-    db_name = var.db_name
-    db_instance = module.database.instance_connection_name
-    gcs_backend = module.artifacts.url
-    module_depends_on = var.module_depends_on
-    consent_screen_support_email = var.consent_screen_support_email
-    web_app_users = var.web_app_users
-    network_short_name = var.network_short_name
+  source                       = "./server"
+  mlflow_server                = var.mlflow_server
+  create_default_service       = var.create_default_service
+  location                     = var.server_location
+  docker_image_name            = var.server_docker_image
+  env_variables                = var.server_env_variables
+  db_private_ip                = module.database.private_ip
+  project_id                   = var.project_id
+  db_password_name             = var.db_password_name
+  db_username                  = var.db_username
+  db_name                      = var.db_name
+  db_instance                  = module.database.instance_connection_name
+  gcs_backend                  = module.artifacts.url
+  module_depends_on            = var.module_depends_on
+  consent_screen_support_email = var.consent_screen_support_email
+  web_app_users                = var.web_app_users
+  network_short_name           = var.network_short_name
+  oauth_client_id              = var.oauth_client_id
+  oauth_client_secret          = var.oauth_client_secret
+  create_brand                 = var.create_brand
+  brand_name                   = var.brand_name
 }

IaC/modules/mlflow/outputs.tf

@@ -0,0 +1,11 @@
+output "app_id" {
+  value = module.server.app_id
+}
+
+output "mlflow_service" {
+  value = module.server.mlflow_service
+}
+
+output "artifacts_bucket_name" {
+  value = module.artifacts.name
+}