diff --git a/.github/workflows/module-4.yaml b/.github/workflows/module-4.yaml index 95f2a21..15fbeb8 100644 --- a/.github/workflows/module-4.yaml +++ b/.github/workflows/module-4.yaml @@ -8,7 +8,7 @@ env: IMAGE_MAIN_TAG: latest jobs: - push-image: + dagster-image: runs-on: ubuntu-latest steps: - name: Checkout diff --git a/module-3/classic-example/Makefile b/module-3/classic-example/Makefile index a0f9698..fcd19ff 100644 --- a/module-3/classic-example/Makefile +++ b/module-3/classic-example/Makefile @@ -1,6 +1,10 @@ build: docker build -f Dockerfile -t classic-example:latest . +push: + docker tag classic-example:latest ghcr.io/kyryl-opens-ml/classic-example:latest + docker push ghcr.io/kyryl-opens-ml/classic-example:latest + run_dev: build docker run -it -v ${PWD}:/main classic-example:latest /bin/bash diff --git a/module-4/PRACTICE.md b/module-4/PRACTICE.md index 4564aaf..5c631d4 100644 --- a/module-4/PRACTICE.md +++ b/module-4/PRACTICE.md @@ -5,8 +5,7 @@ # H7: Kubeflow + AirFlow pipelines -## Reading list: - +## Reading list: - [Kubeflow pipelines Standalone Deployment](https://www.kubeflow.org/docs/components/pipelines/v1/installation/standalone-deployment/) - [Kubeflow Pipelines SDK API Reference](https://kubeflow-pipelines.readthedocs.io/en/) @@ -17,7 +16,6 @@ - [KubernetesPodOperator](https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/operators.html) - ## Task: For this task, you will need both a training and an inference pipeline. The training pipeline should include at least the following steps: Load Training Data, Train Model, Save Trained Models. Additional steps may be added as desired. Similarly, the inference pipeline should include at least the following steps: Load Data for Inference, Load Trained Model, Run Inference, Save Inference Results. You may also add extra steps to this pipeline as needed. @@ -31,12 +29,11 @@ For this task, you will need both a training and an inference pipeline. The trai - PR6: Write an Airflow inference pipeline. -## Criteria: +## Criteria: - 6 PRs merged. - # H8: Dagster ## Reading list: diff --git a/module-4/README.md b/module-4/README.md index bcf9f3c..e248cc4 100644 --- a/module-4/README.md +++ b/module-4/README.md @@ -10,11 +10,11 @@ # Reference implementation -*** +*** # Setup -Create kind cluster +Create kind cluster ```bash kind create cluster --name ml-in-production @@ -28,11 +28,7 @@ k9s -A # Airflow -## Deploy airflow locally - -```bash -export AIRFLOW_HOME=$PWD/airflow_pipelines -``` +Install ```bash AIRFLOW_VERSION=2.9.2 @@ -42,107 +38,93 @@ pip install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL} pip install apache-airflow-providers-cncf-kubernetes==8.3.3 ``` -1. Run standalone airflow +Run standalone airflow -``` +```bash +export AIRFLOW_HOME=./airflow_pipelines export AIRFLOW__CORE__LOAD_EXAMPLES=False +export WANDB_PROJECT=**************** +export WANDB_API_KEY=**************** airflow standalone ``` -2. Create storage +Create storage +```bash +kubectl create -f ./airflow_pipelines/volumes.yaml ``` -kubectl create -f airflow-volumes.yaml -``` - -3. Read to run pipelines - -- https://madewithml.com/courses/mlops/orchestration/ - - -### References: -- https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/operators.html -- https://www.astronomer.io/guides/kubepod-operator/ -- https://www.astronomer.io/guides/airflow-passing-data-between-tasks/ +Open UI +```bash +open http://0.0.0.0:8080 +``` -# Kubeflow pipelines - -## Deploy kubeflow pipelines - -Create directly +Trigger training job. -``` -export PIPELINE_VERSION=2.0.3 -kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=$PIPELINE_VERSION" -kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io -kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/env/dev?ref=$PIPELINE_VERSION" +```bash +airflow dags trigger training_dag ``` -Create yaml and applay with kubectl (better option) +Trigger 5 training jobs. +```bash +for i in {1..5}; do airflow dags trigger training_dag; sleep 1; done ``` -export PIPELINE_VERSION=2.0.3 -kubectl kustomize "github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=$PIPELINE_VERSION" > kfp-yml/res.yaml -kubectl kustomize "github.com/kubeflow/pipelines/manifests/kustomize/env/dev?ref=$PIPELINE_VERSION" > kfp-yml/pipelines.yaml -kubectl create -f kfp-yml/res.yaml -kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io -kubectl create -f kfp-yml/pipelines.yaml +Trigger inference job. + +```bash +airflow dags trigger inference_dag ``` +Trigger 5 inference jobs. -Access UI and minio +```bash +for i in {1..5}; do airflow dags trigger inference_dag; sleep 1; done +``` +### References: -``` -kubectl port-forward --address=0.0.0.0 svc/minio-service 9000:9000 -n kubeflow -kubectl port-forward --address=0.0.0.0 svc/ml-pipeline-ui 8888:80 -n kubeflow -``` +- [AI + ML examples of DAGs](https://registry.astronomer.io/dags?categoryName=AI+%2B+Machine+Learning&limit=24&sorts=updatedAt%3Adesc) +- [Pass data between tasks](https://www.astronomer.io/docs/learn/airflow-passing-data-between-tasks) -## Create pipelines +# Kubeflow pipelines -Setup env variables +Install -``` +```bash export WANDB_PROJECT=**************** export WANDB_API_KEY=**************** +export PIPELINE_VERSION=2.2.0 +kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/cluster-scoped-resources?ref=$PIPELINE_VERSION" +kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io +kubectl apply -k "github.com/kubeflow/pipelines/manifests/kustomize/env/dev?ref=$PIPELINE_VERSION" ``` -### Training & Inference V2 (2.0.3) - -``` -python kfp-training-pipeline_v2.py http://0.0.0.0:8080 -``` +Access UI and minio -``` -python kfp-inference-pipeline_v2.py http://0.0.0.0:8080 +```bash +kubectl port-forward --address=0.0.0.0 svc/minio-service 9000:9000 -n kubeflow +kubectl port-forward --address=0.0.0.0 svc/ml-pipeline-ui 3000:80 -n kubeflow ``` +Create training job. -### Training & Inference V1 (1.8.9) +```bash +python ./kubeflow_pipelines/kfp_training_pipeline.py http://0.0.0.0:3000 +``` +Create inference job. -``` -python kfp-training-pipeline_v1.py http://0.0.0.0:8080 +```bash +python kubeflow_pipelines/kfp_inference_pipeline.py http://0.0.0.0:3000 ``` -``` -python kfp-inference-pipeline_v1.py http://0.0.0.0:8080 -``` ### References -- https://www.kubeflow.org/docs/components/pipelines/v2/data-types/artifacts/#new-pythonic-artifact-syntax - - - -# Dagster - - -- https://github.com/dagster-io/dagster_llm_finetune -- https://dagster.io/blog/finetuning-llms - +- [Create, use, pass, and track ML artifacts](https://www.kubeflow.org/docs/components/pipelines/v2/data-types/artifacts/#new-pythonic-artifact-syntax) +- [Vertex AI](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) diff --git a/module-4/airflow_pipelines/.gitignore b/module-4/airflow_pipelines/.gitignore index 024bec1..595e758 100644 --- a/module-4/airflow_pipelines/.gitignore +++ b/module-4/airflow_pipelines/.gitignore @@ -2,4 +2,5 @@ logs airflow.cfg airflow.db standalone_admin_password.txt -webserver_config.py \ No newline at end of file +webserver_config.py +airflow-webserver.pid \ No newline at end of file diff --git a/module-4/airflow_pipelines/dags/inference_dag.py b/module-4/airflow_pipelines/dags/inference_dag.py index c2e3b12..a009407 100644 --- a/module-4/airflow_pipelines/dags/inference_dag.py +++ b/module-4/airflow_pipelines/dags/inference_dag.py @@ -1,44 +1,69 @@ +import os from datetime import datetime from airflow import DAG from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from kubernetes.client import models as k8s +DOCKER_IMAGE = "ghcr.io/kyryl-opens-ml/classic-example:main" +STORAGE_NAME = "training-storage" +WANDB_PROJECT = os.getenv("WANDB_PROJECT") +WANDB_API_KEY = os.getenv("WANDB_API_KEY") + volume = k8s.V1Volume( name="inference-storage", - persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource(claim_name="inference-storage"), + persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource( + claim_name="inference-storage" + ), +) +volume_mount = k8s.V1VolumeMount( + name="inference-storage", mount_path="/tmp/", sub_path=None ) -volume_mount = k8s.V1VolumeMount(name="inference-storage", mount_path="/tmp/", sub_path=None) -with DAG(start_date=datetime(2021, 1, 1), catchup=False, schedule_interval=None, dag_id="inference_dag") as dag: +with DAG( + start_date=datetime(2021, 1, 1), + catchup=False, + schedule_interval=None, + dag_id="inference_dag", +) as dag: clean_storage_before_start = KubernetesPodOperator( name="clean_storage_before_start", - image="kyrylprojector/nlp-sample:latest", + image=DOCKER_IMAGE, cmds=["rm", "-rf", "/tmp/data/*"], task_id="clean_storage_before_start", in_cluster=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], ) load_data = KubernetesPodOperator( name="load_data", - image="kyrylprojector/nlp-sample:latest", - cmds=["python", "nlp_sample/cli.py", "load-cola-data", "/tmp/data/"], + image=DOCKER_IMAGE, + cmds=["python", "classic_example/cli.py", "load-sst2-data", "/tmp/data/"], task_id="load_data", in_cluster=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], ) load_model = KubernetesPodOperator( name="load_model", - image="kyrylprojector/nlp-sample:latest", - cmds=["python", "nlp_sample/cli.py", "load-from-registry", "kfp-pipeline:latest", "/tmp/results/"], + image=DOCKER_IMAGE, + cmds=[ + "python", + "classic_example/cli.py", + "load-from-registry", + "airflow-pipeline:latest", + "/tmp/results/", + ], task_id="load_model", - env_vars={"WANDB_PROJECT": "course-27-10-2023-week-3", "WANDB_API_KEY": ""}, + env_vars={"WANDB_PROJECT": WANDB_PROJECT, "WANDB_API_KEY": WANDB_API_KEY}, in_cluster=False, namespace="default", volumes=[volume], @@ -47,10 +72,10 @@ run_inference = KubernetesPodOperator( name="run_inference", - image="kyrylprojector/nlp-sample:latest", + image=DOCKER_IMAGE, cmds=[ "python", - "nlp_sample/cli.py", + "classic_example/cli.py", "run-inference-on-dataframe", "/tmp/data/test.csv", "/tmp/results/", @@ -59,17 +84,21 @@ task_id="run_inference", in_cluster=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], ) clean_up = KubernetesPodOperator( name="clean_up", - image="kyrylprojector/nlp-sample:latest", + image=DOCKER_IMAGE, cmds=["rm", "-rf", "/tmp/data/*"], task_id="clean_up", in_cluster=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], trigger_rule="all_done", diff --git a/module-4/airflow_pipelines/dags/training_dag.py b/module-4/airflow_pipelines/dags/training_dag.py index 515a686..0197734 100644 --- a/module-4/airflow_pipelines/dags/training_dag.py +++ b/module-4/airflow_pipelines/dags/training_dag.py @@ -1,72 +1,106 @@ +import os from datetime import datetime from airflow import DAG from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator from kubernetes.client import models as k8s +DOCKER_IMAGE = "ghcr.io/kyryl-opens-ml/classic-example:main" +STORAGE_NAME = "training-storage" +WANDB_PROJECT = os.getenv("WANDB_PROJECT") +WANDB_API_KEY = os.getenv("WANDB_API_KEY") + volume = k8s.V1Volume( - name="training-storage", - persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource(claim_name="training-storage"), + name=STORAGE_NAME, + persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource( + claim_name=STORAGE_NAME + ), ) -volume_mount = k8s.V1VolumeMount(name="training-storage", mount_path="/tmp/", sub_path=None) +volume_mount = k8s.V1VolumeMount(name=STORAGE_NAME, mount_path="/tmp/", sub_path=None) -with DAG(start_date=datetime(2021, 1, 1), catchup=False, schedule_interval=None, dag_id="training_dag") as dag: +with DAG( + start_date=datetime(2021, 1, 1), + catchup=False, + schedule_interval=None, + dag_id="training_dag", +) as dag: clean_storage_before_start = KubernetesPodOperator( name="clean_storage_before_start", - image="kyrylprojector/nlp-sample:latest", + image=DOCKER_IMAGE, cmds=["rm", "-rf", "/tmp/*"], task_id="clean_storage_before_start", is_delete_operator_pod=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], ) load_data = KubernetesPodOperator( name="load_data", - image="kyrylprojector/nlp-sample:latest", - cmds=["python", "nlp_sample/cli.py", "load-cola-data", "/tmp/data/"], + image=DOCKER_IMAGE, + cmds=["python", "classic_example/cli.py", "load-sst2-data", "/tmp/data/"], task_id="load_data", in_cluster=False, is_delete_operator_pod=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], ) train_model = KubernetesPodOperator( name="train_model", - image="kyrylprojector/nlp-sample:latest", - cmds=["python", "nlp_sample/cli.py", "train", "tests/data/test_config.json"], + image=DOCKER_IMAGE, + cmds=[ + "python", + "classic_example/cli.py", + "train", + "tests/data/test_config.json", + ], task_id="train_model", in_cluster=False, is_delete_operator_pod=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], ) upload_model = KubernetesPodOperator( name="upload_model", - image="kyrylprojector/nlp-sample:latest", - cmds=["python", "nlp_sample/cli.py", "upload-to-registry", "airflow-pipeline", "/tmp/results"], + image=DOCKER_IMAGE, + cmds=[ + "python", + "classic_example/cli.py", + "upload-to-registry", + "airflow-pipeline", + "/tmp/results", + ], task_id="upload_model", - env_vars={"WANDB_PROJECT": "course-27-10-2023-week-3", "WANDB_API_KEY": ""}, + env_vars={"WANDB_PROJECT": WANDB_PROJECT, "WANDB_API_KEY": WANDB_API_KEY}, in_cluster=False, is_delete_operator_pod=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], ) clean_up = KubernetesPodOperator( name="clean_up", - image="kyrylprojector/nlp-sample:latest", + image=DOCKER_IMAGE, cmds=["rm", "-rf", "/tmp/*"], task_id="clean_up", in_cluster=False, is_delete_operator_pod=False, namespace="default", + startup_timeout_seconds=600, + image_pull_policy="Always", volumes=[volume], volume_mounts=[volume_mount], trigger_rule="all_done", diff --git a/module-4/airflow_pipelines/airflow-volumes.yaml b/module-4/airflow_pipelines/volumes.yaml similarity index 91% rename from module-4/airflow_pipelines/airflow-volumes.yaml rename to module-4/airflow_pipelines/volumes.yaml index 1fad40c..dbff02c 100644 --- a/module-4/airflow_pipelines/airflow-volumes.yaml +++ b/module-4/airflow_pipelines/volumes.yaml @@ -19,7 +19,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - ml-in-production-course-week-4-control-plane + - ml-in-production-control-plane --- apiVersion: v1 kind: PersistentVolumeClaim @@ -53,7 +53,7 @@ spec: - key: kubernetes.io/hostname operator: In values: - - ml-in-production-course-week-4-control-plane + - ml-in-production-control-plane --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/module-4/dagster_pipelines/Dockerfile b/module-4/dagster_pipelines/Dockerfile new file mode 100644 index 0000000..4ba6157 --- /dev/null +++ b/module-4/dagster_pipelines/Dockerfile @@ -0,0 +1,17 @@ +FROM huggingface/transformers-pytorch-gpu:4.35.2 + +WORKDIR /app + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt +RUN MAX_JOBS=4 pip install flash-attn==2.5.7 --no-build-isolation + +RUN ln -s /usr/bin/python3 /usr/bin/python + +ENV PYTHONPATH /app +COPY . . + +CMD [ "bash" ] \ No newline at end of file diff --git a/module-4/dagster_pipelines/pipeline.py b/module-4/dagster_pipelines/pipeline.py new file mode 100644 index 0000000..91cd3fc --- /dev/null +++ b/module-4/dagster_pipelines/pipeline.py @@ -0,0 +1,340 @@ +import logging +from functools import partial +from pathlib import Path + +import torch +from datasets import Dataset, DatasetDict +from generative_example.config import DataTrainingArguments, ModelArguments +from generative_example.utils import setup_logger +from peft import LoraConfig, TaskType +from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + HfArgumentParser, + TrainingArguments, + set_seed, +) +from trl import SFTTrainer + +logger = logging.getLogger(__name__) + + +def create_message_column(row): + messages = [] + user = {"content": f"{row['context']}\n Input: {row['question']}", "role": "user"} + messages.append(user) + assistant = {"content": f"{row['answer']}", "role": "assistant"} + messages.append(assistant) + return {"messages": messages} + + +def format_dataset_chatml(row, tokenizer): + return { + "text": tokenizer.apply_chat_template( + row["messages"], add_generation_prompt=False, tokenize=False + ) + } + + +def process_dataset(model_id: str, train_file: str, test_file: str) -> DatasetDict: + dataset = DatasetDict( + { + "train": Dataset.from_json(train_file), + "test": Dataset.from_json(test_file), + } + ) + + tokenizer_id = model_id + tokenizer = AutoTokenizer.from_pretrained(tokenizer_id) + tokenizer.padding_side = "right" + + dataset_chatml = dataset.map(create_message_column) + dataset_chatml = dataset_chatml.map( + partial(format_dataset_chatml, tokenizer=tokenizer) + ) + return dataset_chatml + + +def get_model(model_id: str, device_map): + if torch.cuda.is_bf16_supported(): + compute_dtype = torch.bfloat16 + attn_implementation = "flash_attention_2" + # If bfloat16 is not supported, 'compute_dtype' is set to 'torch.float16' and 'attn_implementation' is set to 'sdpa'. + else: + compute_dtype = torch.float16 + attn_implementation = "sdpa" + + # This line of code is used to print the value of 'attn_implementation', which indicates the chosen attention implementation. + print(attn_implementation) + + tokenizer = AutoTokenizer.from_pretrained( + model_id, trust_remote_code=True, add_eos_token=True, use_fast=True + ) + tokenizer.pad_token = tokenizer.unk_token + tokenizer.pad_token_id = tokenizer.convert_tokens_to_ids(tokenizer.pad_token) + tokenizer.padding_side = "left" + + model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=compute_dtype, + trust_remote_code=True, + device_map=device_map, + attn_implementation=attn_implementation, + ) + return tokenizer, model + + +def get_config(config_path: Path): + parser = HfArgumentParser( + (ModelArguments, DataTrainingArguments, TrainingArguments) + ) + model_args, data_args, training_args = parser.parse_json_file(config_path) + return model_args, data_args, training_args + + +def train(config_path: Path): + setup_logger(logger) + + model_args, data_args, training_args = get_config(config_path=config_path) + + logger.info(f"model_args = {model_args}") + logger.info(f"data_args = {data_args}") + logger.info(f"training_args = {training_args}") + + device_map = {"": 0} + target_modules = [ + "k_proj", + "q_proj", + "v_proj", + "o_proj", + "gate_proj", + "down_proj", + "up_proj", + ] + + set_seed(training_args.seed) + + dataset_chatml = process_dataset( + model_id=model_args.model_id, + train_file=data_args.train_file, + test_file=data_args.test_file, + ) + logger.info(dataset_chatml["train"][0]) + + tokenizer, model = get_model(model_id=model_args.model_id, device_map=device_map) + peft_config = LoraConfig( + r=model_args.lora_r, + lora_alpha=model_args.lora_alpha, + lora_dropout=model_args.lora_dropout, + task_type=TaskType.CAUSAL_LM, + target_modules=target_modules, + ) + + trainer = SFTTrainer( + model=model, + train_dataset=dataset_chatml["train"], + eval_dataset=dataset_chatml["test"], + peft_config=peft_config, + dataset_text_field="text", + max_seq_length=512, + tokenizer=tokenizer, + args=training_args, + ) + + trainer.train() + trainer.save_model() + trainer.create_model_card() + + +import logging +import sys +from pathlib import Path + +import datasets +import transformers +import wandb + + +def setup_logger(logger): + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + log_level = "INFO" + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + +def upload_to_registry(model_name: str, model_path: Path): + with wandb.init() as _: + art = wandb.Artifact(model_name, type="model") + art.add_file(model_path / "README.md") + art.add_file(model_path / "adapter_config.json") + art.add_file(model_path / "adapter_model.safetensors") + art.add_file(model_path / "special_tokens_map.json") + art.add_file(model_path / "tokenizer.json") + art.add_file(model_path / "tokenizer_config.json") + art.add_file(model_path / "training_args.bin") + wandb.log_artifact(art) + + +def load_from_registry(model_name: str, model_path: Path): + with wandb.init() as run: + artifact = run.use_artifact(model_name, type="model") + artifact_dir = artifact.download(root=model_path) + print(f"{artifact_dir}") + + +import json +import logging +from pathlib import Path + +import evaluate +from peft import AutoPeftModelForCausalLM +from tqdm import tqdm +from transformers import pipeline + +logger = logging.getLogger() + + +class Predictor: + def __init__(self, model_load_path: str): + device_map = {"": 0} + new_model = AutoPeftModelForCausalLM.from_pretrained( + model_load_path, + low_cpu_mem_usage=True, + return_dict=True, + torch_dtype=torch.bfloat16, + trust_remote_code=True, + device_map=device_map, + ) + merged_model = new_model.merge_and_unload() + + tokenizer = AutoTokenizer.from_pretrained( + model_load_path, trust_remote_code=True + ) + pipe = pipeline("text-generation", model=merged_model, tokenizer=tokenizer) + self.pipe = pipe + + @torch.no_grad() + def predict(self, question: str, context: str) -> str: + pipe = self.pipe + + messages = [{"content": f"{context}\n Input: {question}", "role": "user"}] + + prompt = pipe.tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + outputs = pipe( + prompt, + max_new_tokens=256, + do_sample=True, + num_beams=1, + temperature=0.3, + top_k=50, + top_p=0.95, + max_time=180, + ) + sql = outputs[0]["generated_text"][len(prompt) :].strip() + return sql + + +def run_inference_on_json(json_path: Path, model_load_path: Path, result_path: Path): + df = Dataset.from_json(str(json_path)).to_pandas() + model = Predictor(model_load_path=model_load_path) + + generated_sql = [] + for idx in tqdm(range(len(df))): + context = df.iloc[idx]["context"] + question = df.iloc[idx]["question"] + + sql = model.predict(question=question, context=context) + generated_sql.append(sql) + df["generated_sql"] = generated_sql + df.to_csv(result_path, index=False) + + +def run_evaluate_on_json(json_path: Path, model_load_path: Path, result_path: Path): + df = Dataset.from_json(str(json_path)).to_pandas() + model = Predictor(model_load_path=model_load_path) + + generated_sql = [] + for idx in tqdm(range(len(df))): + context = df.iloc[idx]["context"] + question = df.iloc[idx]["question"] + + sql = model.predict(question=question, context=context) + generated_sql.append(sql) + + gt_sql = df["answer"].values + rouge = evaluate.load("rouge") + results = rouge.compute(predictions=generated_sql, references=gt_sql) + print(f"Metrics {results}") + with open(result_path, "w") as f: + json.dump(results, f) + + +from pathlib import Path +from random import randrange + +from datasets import DatasetDict, load_dataset + + +def _get_sql_data(random_state: int = 42, subsample: float = None) -> DatasetDict: + dataset_name = "b-mc2/sql-create-context" + dataset = load_dataset(dataset_name, split="train") + print(f"dataset size: {len(dataset)}") + print(dataset[randrange(len(dataset))]) + + if subsample is not None: + dataset = dataset.shuffle(seed=random_state).select( + range(int(len(dataset) * subsample)) + ) + print(f"dataset new size: {len(dataset)}") + + datasets = dataset.train_test_split(test_size=0.05, seed=random_state) + return datasets + + +def load_sql_data(path_to_save: Path, subsample: float = None): + path_to_save.mkdir(parents=True, exist_ok=True) + + datasets = _get_sql_data(subsample=subsample) + + datasets["train"].to_json(path_to_save / "train.json") + datasets["test"].to_json(path_to_save / "test.json") + + +def load_sql_data_file_input( + path_to_train: Path, path_to_test: Path, subsample: float = None +): + path_to_train.parent.mkdir(parents=True, exist_ok=True) + path_to_test.parent.mkdir(parents=True, exist_ok=True) + + datasets = _get_sql_data(subsample=subsample) + + datasets["train"].to_json(path_to_train) + datasets["test"].to_json(path_to_test) + + +from dataclasses import dataclass + + +@dataclass +class DataTrainingArguments: + train_file: str + test_file: str + + +@dataclass +class ModelArguments: + model_id: str + lora_r: int + lora_alpha: int + lora_dropout: float diff --git a/module-4/kfp-yml/pipelines.yaml b/module-4/kfp-yml/pipelines.yaml deleted file mode 100644 index b0e9562..0000000 --- a/module-4/kfp-yml/pipelines.yaml +++ /dev/null @@ -1,2238 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: application - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - internal.kpt.dev/upstream-identifier: '|ServiceAccount|default|argo' - labels: - application-crd-id: kubeflow-pipelines - name: argo - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: cache-server - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-cache - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-container-builder - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-metadata-writer - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-viewer - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: metadata-grpc-server - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-persistenceagent - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-scheduledworkflow - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-viewer-crd-service-account - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-visualizationserver - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: mysql - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: pipeline-runner - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: proxy-agent-runner - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: application-manager-role - namespace: kubeflow -rules: -- apiGroups: - - '*' - resources: - - '*' - verbs: - - get - - list - - update - - patch - - watch -- apiGroups: - - app.k8s.io - resources: - - '*' - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - annotations: - internal.kpt.dev/upstream-identifier: rbac.authorization.k8s.io|Role|default|argo-role - labels: - application-crd-id: kubeflow-pipelines - name: argo-role - namespace: kubeflow -rules: -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - pods - - pods/exec - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - watch - - list -- apiGroups: - - "" - resources: - - persistentvolumeclaims - - persistentvolumeclaims/finalizers - verbs: - - create - - update - - delete - - get -- apiGroups: - - argoproj.io - resources: - - workflows - - workflows/finalizers - - workflowtasksets - - workflowtasksets/finalizers - verbs: - - get - - list - - watch - - update - - patch - - delete - - create -- apiGroups: - - argoproj.io - resources: - - workflowtemplates - - workflowtemplates/finalizers - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - get - - list -- apiGroups: - - argoproj.io - resources: - - workflowtaskresults - verbs: - - list - - watch - - deletecollection -- apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - get - - list -- apiGroups: - - "" - resources: - - secrets - verbs: - - get -- apiGroups: - - argoproj.io - resources: - - cronworkflows - - cronworkflows/finalizers - verbs: - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - policy - resources: - - poddisruptionbudgets - verbs: - - create - - get - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: cache-deployer - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-cache-deployer-role - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - delete - - get - - patch - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: cache-server - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-cache-role - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - get - - list - - watch - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: kubeflow-pipelines-metadata-writer-role - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-metadata-writer-role - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - get - - list - - watch - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - pods - - pods/log - verbs: - - get - - list - - delete -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - kubeflow.org - resources: - - scheduledworkflows - verbs: - - create - - get - - list - - update - - patch - - delete -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-persistenceagent-role - namespace: kubeflow -rules: -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - get - - list - - watch -- apiGroups: - - kubeflow.org - resources: - - scheduledworkflows - verbs: - - get - - list - - watch -- apiGroups: - - pipelines.kubeflow.org - resources: - - scheduledworkflows - - workflows - verbs: - - report -- apiGroups: - - pipelines.kubeflow.org - resources: - - runs - verbs: - - reportMetrics - - readArtifact ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: ml-pipeline-scheduledworkflow-role - application-crd-id: kubeflow-pipelines - name: ml-pipeline-scheduledworkflow-role - namespace: kubeflow -rules: -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - kubeflow.org - resources: - - scheduledworkflows - - scheduledworkflows/finalizers - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - pods - - pods/log - verbs: - - get -- apiGroups: - - "" - resources: - - events - verbs: - - list -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list -- apiGroups: - - kubeflow.org - resources: - - viewers - verbs: - - create - - get - - list - - watch - - delete -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - get - - list ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-viewer-controller-role - namespace: kubeflow -rules: -- apiGroups: - - '*' - resources: - - deployments - - services - verbs: - - create - - get - - list - - watch - - update - - patch - - delete -- apiGroups: - - kubeflow.org - resources: - - viewers - - viewers/finalizers - verbs: - - create - - get - - list - - watch - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: pipeline-runner - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - verbs: - - get - - watch - - list -- apiGroups: - - "" - resources: - - persistentvolumes - - persistentvolumeclaims - verbs: - - '*' -- apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshots - verbs: - - create - - delete - - get -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - get - - list - - watch - - update - - patch -- apiGroups: - - "" - resources: - - pods - - pods/exec - - pods/log - - services - verbs: - - '*' -- apiGroups: - - "" - - apps - - extensions - resources: - - deployments - - replicasets - verbs: - - '*' -- apiGroups: - - kubeflow.org - resources: - - '*' - verbs: - - '*' -- apiGroups: - - batch - resources: - - jobs - verbs: - - '*' -- apiGroups: - - machinelearning.seldon.io - resources: - - seldondeployments - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app: proxy-agent-runner - application-crd-id: kubeflow-pipelines - name: proxy-agent-runner - namespace: kubeflow -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: application-manager-rolebinding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: application-manager-role -subjects: -- kind: ServiceAccount - name: application - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - annotations: - internal.kpt.dev/upstream-identifier: rbac.authorization.k8s.io|RoleBinding|default|argo-binding - labels: - application-crd-id: kubeflow-pipelines - name: argo-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: argo-role -subjects: -- kind: ServiceAccount - name: argo - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app: cache-server - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-cache-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubeflow-pipelines-cache-role -subjects: -- kind: ServiceAccount - name: kubeflow-pipelines-cache - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app: cache-deployer - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-cache-deployer-rolebinding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubeflow-pipelines-cache-deployer-role -subjects: -- kind: ServiceAccount - name: kubeflow-pipelines-cache-deployer-sa ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: kubeflow-pipelines-metadata-writer-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubeflow-pipelines-metadata-writer-role -subjects: -- kind: ServiceAccount - name: kubeflow-pipelines-metadata-writer - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ml-pipeline -subjects: -- kind: ServiceAccount - name: ml-pipeline - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-persistenceagent-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ml-pipeline-persistenceagent-role -subjects: -- kind: ServiceAccount - name: ml-pipeline-persistenceagent - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-scheduledworkflow-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ml-pipeline-scheduledworkflow-role -subjects: -- kind: ServiceAccount - name: ml-pipeline-scheduledworkflow - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ml-pipeline-ui -subjects: -- kind: ServiceAccount - name: ml-pipeline-ui - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-viewer-crd-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: ml-pipeline-viewer-controller-role -subjects: -- kind: ServiceAccount - name: ml-pipeline-viewer-crd-service-account - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: pipeline-runner-binding - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: pipeline-runner -subjects: -- kind: ServiceAccount - name: pipeline-runner - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app: proxy-agent-runner - application-crd-id: kubeflow-pipelines - name: proxy-agent-runner - namespace: kubeflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: proxy-agent-runner -subjects: -- kind: ServiceAccount - name: proxy-agent-runner - namespace: kubeflow ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: inverse-proxy-config - namespace: kubeflow ---- -apiVersion: v1 -data: - defaultPipelineRoot: "" -kind: ConfigMap -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: kfp-launcher - namespace: kubeflow ---- -apiVersion: v1 -data: - METADATA_GRPC_SERVICE_HOST: metadata-grpc-service - METADATA_GRPC_SERVICE_PORT: "8080" -kind: ConfigMap -metadata: - labels: - application-crd-id: kubeflow-pipelines - component: metadata-grpc-server - name: metadata-grpc-configmap - namespace: kubeflow ---- -apiVersion: v1 -data: - viewer-pod-template.json: |- - { - "spec": { - "serviceAccountName": "kubeflow-pipelines-viewer" - } - } -kind: ConfigMap -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui-configmap - namespace: kubeflow ---- -apiVersion: v1 -data: - ConMaxLifeTime: 120s - DEFAULT_CACHE_STALENESS: "" - MAXIMUM_CACHE_STALENESS: "" - appName: pipeline - appVersion: 2.0.3 - autoUpdatePipelineDefaultVersion: "true" - bucketName: mlpipeline - cacheDb: cachedb - cacheImage: gcr.io/google-containers/busybox - cacheNodeRestrictions: "false" - cronScheduleTimezone: UTC - dbHost: mysql - dbPort: "3306" - dbType: mysql - defaultPipelineRoot: "" - mlmdDb: metadb - mysqlHost: mysql - mysqlPort: "3306" - pipelineDb: mlpipeline - warning: | - 1. Do not use kubectl to edit this configmap, because some values are used - during kustomize build. Instead, change the configmap and apply the entire - kustomize manifests again. - 2. After updating the configmap, some deployments may need to be restarted - until the changes take effect. A quick way to restart all deployments in a - namespace: `kubectl rollout restart deployment -n `. -kind: ConfigMap -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: pipeline-install-config - namespace: kubeflow ---- -apiVersion: v1 -data: - artifactRepository: | - archiveLogs: true - s3: - endpoint: "minio-service.kubeflow:9000" - bucket: "mlpipeline" - # keyFormat is a format pattern to define how artifacts will be organized in a bucket. - # It can reference workflow metadata variables such as workflow.namespace, workflow.name, - # pod.name. Can also use strftime formating of workflow.creationTimestamp so that workflow - # artifacts can be organized by date. If omitted, will use `{{workflow.name}}/{{pod.name}}`, - # which has potential for have collisions, because names do not guarantee they are unique - # over the lifetime of the cluster. - # Refer to https://kubernetes.io/docs/concepts/overview/working-with-objects/names/. - # - # The following format looks like: - # artifacts/my-workflow-abc123/2018/08/23/my-workflow-abc123-1234567890 - # Adding date into the path greatly reduces the chance of {{pod.name}} collision. - keyFormat: "artifacts/{{workflow.name}}/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{pod.name}}" - # insecure will disable TLS. Primarily used for minio installs not configured with TLS - insecure: true - accessKeySecret: - name: mlpipeline-minio-artifact - key: accesskey - secretKeySecret: - name: mlpipeline-minio-artifact - key: secretkey - containerRuntimeExecutor: emissary - executor: | - imagePullPolicy: IfNotPresent -kind: ConfigMap -metadata: - annotations: - internal.kpt.dev/upstream-identifier: '|ConfigMap|default|workflow-controller-configmap' - labels: - application-crd-id: kubeflow-pipelines - name: workflow-controller-configmap - namespace: kubeflow ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: mlpipeline-minio-artifact - namespace: kubeflow -stringData: - accesskey: minio - secretkey: minio123 ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: mysql-secret - namespace: kubeflow -stringData: - password: "" - username: root ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: cache-server - application-crd-id: kubeflow-pipelines - name: cache-server - namespace: kubeflow -spec: - ports: - - port: 443 - targetPort: webhook-api - selector: - app: cache-server - application-crd-id: kubeflow-pipelines ---- -apiVersion: v1 -kind: Service -metadata: - labels: - application-crd-id: kubeflow-pipelines - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: controller-manager-service - namespace: kubeflow -spec: - ports: - - port: 443 - selector: - application-crd-id: kubeflow-pipelines - control-plane: controller-manager - controller-tools.k8s.io: "1.0" ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: metadata-envoy - application-crd-id: kubeflow-pipelines - name: metadata-envoy-service - namespace: kubeflow -spec: - ports: - - name: md-envoy - port: 9090 - protocol: TCP - selector: - application-crd-id: kubeflow-pipelines - component: metadata-envoy - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: metadata - application-crd-id: kubeflow-pipelines - name: metadata-grpc-service - namespace: kubeflow -spec: - ports: - - name: grpc-api - port: 8080 - protocol: TCP - selector: - application-crd-id: kubeflow-pipelines - component: metadata-grpc-server - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: minio-service - namespace: kubeflow -spec: - ports: - - name: http - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - application-crd-id: kubeflow-pipelines ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - prometheus.io/port: "8888" - prometheus.io/scheme: http - prometheus.io/scrape: "true" - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow -spec: - ports: - - name: http - port: 8888 - protocol: TCP - targetPort: 8888 - - name: grpc - port: 8887 - protocol: TCP - targetPort: 8887 - selector: - app: ml-pipeline - application-crd-id: kubeflow-pipelines ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui - namespace: kubeflow -spec: - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 3000 - selector: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines ---- -apiVersion: v1 -kind: Service -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: ml-pipeline-visualizationserver - namespace: kubeflow -spec: - ports: - - name: http - port: 8888 - protocol: TCP - targetPort: 8888 - selector: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines ---- -apiVersion: v1 -kind: Service -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: mysql - namespace: kubeflow -spec: - ports: - - port: 3306 - protocol: TCP - targetPort: 3306 - selector: - app: mysql - application-crd-id: kubeflow-pipelines ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - internal.kpt.dev/upstream-identifier: '|Service|default|workflow-controller-metrics' - workflows.argoproj.io/description: | - This service is deprecated. It will be removed in v3.4. - - https://github.com/argoproj/argo-workflows/issues/8441 - labels: - app: workflow-controller - application-crd-id: kubeflow-pipelines - name: workflow-controller-metrics - namespace: kubeflow -spec: - ports: - - name: metrics - port: 9090 - protocol: TCP - targetPort: 9090 - selector: - app: workflow-controller - application-crd-id: kubeflow-pipelines ---- -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - annotations: - internal.kpt.dev/upstream-identifier: scheduling.k8s.io|PriorityClass|default|workflow-controller - labels: - application-crd-id: kubeflow-pipelines - name: workflow-controller -value: 1000000 ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: minio-pvc - namespace: kubeflow -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - labels: - application-crd-id: kubeflow-pipelines - name: mysql-pv-claim - namespace: kubeflow -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 20Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: cache-deployer - application-crd-id: kubeflow-pipelines - name: cache-deployer-deployment - namespace: kubeflow -spec: - replicas: 1 - selector: - matchLabels: - app: cache-deployer - application-crd-id: kubeflow-pipelines - strategy: - type: Recreate - template: - metadata: - labels: - app: cache-deployer - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: gcr.io/ml-pipeline/cache-deployer:2.0.3 - imagePullPolicy: Always - name: main - restartPolicy: Always - serviceAccountName: kubeflow-pipelines-cache-deployer-sa ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: cache-server - application-crd-id: kubeflow-pipelines - name: cache-server - namespace: kubeflow -spec: - replicas: 1 - selector: - matchLabels: - app: cache-server - application-crd-id: kubeflow-pipelines - template: - metadata: - labels: - app: cache-server - application-crd-id: kubeflow-pipelines - spec: - containers: - - args: - - --db_driver=$(DBCONFIG_DRIVER) - - --db_host=$(DBCONFIG_HOST_NAME) - - --db_port=$(DBCONFIG_PORT) - - --db_name=$(DBCONFIG_DB_NAME) - - --db_user=$(DBCONFIG_USER) - - --db_password=$(DBCONFIG_PASSWORD) - - --namespace_to_watch=$(NAMESPACE_TO_WATCH) - - --listen_port=$(WEBHOOK_PORT) - env: - - name: DEFAULT_CACHE_STALENESS - valueFrom: - configMapKeyRef: - key: DEFAULT_CACHE_STALENESS - name: pipeline-install-config - - name: MAXIMUM_CACHE_STALENESS - valueFrom: - configMapKeyRef: - key: MAXIMUM_CACHE_STALENESS - name: pipeline-install-config - - name: CACHE_IMAGE - valueFrom: - configMapKeyRef: - key: cacheImage - name: pipeline-install-config - - name: CACHE_NODE_RESTRICTIONS - valueFrom: - configMapKeyRef: - key: cacheNodeRestrictions - name: pipeline-install-config - - name: DBCONFIG_DRIVER - value: mysql - - name: DBCONFIG_DB_NAME - valueFrom: - configMapKeyRef: - key: cacheDb - name: pipeline-install-config - - name: DBCONFIG_HOST_NAME - valueFrom: - configMapKeyRef: - key: dbHost - name: pipeline-install-config - - name: DBCONFIG_PORT - valueFrom: - configMapKeyRef: - key: dbPort - name: pipeline-install-config - - name: DBCONFIG_USER - valueFrom: - secretKeyRef: - key: username - name: mysql-secret - - name: DBCONFIG_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: mysql-secret - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: WEBHOOK_PORT - value: "8443" - image: gcr.io/ml-pipeline/cache-server:2.0.3 - imagePullPolicy: Always - name: server - ports: - - containerPort: 8443 - name: webhook-api - volumeMounts: - - mountPath: /etc/webhook/certs - name: webhook-tls-certs - readOnly: true - serviceAccountName: kubeflow-pipelines-cache - volumes: - - name: webhook-tls-certs - secret: - secretName: webhook-server-tls ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - application-crd-id: kubeflow-pipelines - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: controller-manager - namespace: kubeflow -spec: - selector: - matchLabels: - application-crd-id: kubeflow-pipelines - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - template: - metadata: - labels: - application-crd-id: kubeflow-pipelines - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - spec: - containers: - - command: - - /bin/sh - - -c - - | - echo "logs are hidden because volume is too excessive" && - /root/manager 2> /dev/null - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: gcr.io/ml-pipeline/application-crd-controller:1.0-beta-non-cluster-role - imagePullPolicy: IfNotPresent - name: manager - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi - serviceAccountName: application ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - application-crd-id: kubeflow-pipelines - component: metadata-envoy - name: metadata-envoy-deployment - namespace: kubeflow -spec: - replicas: 1 - selector: - matchLabels: - application-crd-id: kubeflow-pipelines - component: metadata-envoy - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - application-crd-id: kubeflow-pipelines - component: metadata-envoy - spec: - containers: - - image: gcr.io/ml-pipeline/metadata-envoy:2.0.3 - name: container - ports: - - containerPort: 9090 - name: md-envoy - - containerPort: 9901 - name: envoy-admin ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - application-crd-id: kubeflow-pipelines - component: metadata-grpc-server - name: metadata-grpc-deployment - namespace: kubeflow -spec: - replicas: 1 - selector: - matchLabels: - application-crd-id: kubeflow-pipelines - component: metadata-grpc-server - template: - metadata: - labels: - application-crd-id: kubeflow-pipelines - component: metadata-grpc-server - spec: - containers: - - args: - - --grpc_port=8080 - - --mysql_config_database=$(MYSQL_DATABASE) - - --mysql_config_host=$(MYSQL_HOST) - - --mysql_config_port=$(MYSQL_PORT) - - --mysql_config_user=$(DBCONFIG_USER) - - --mysql_config_password=$(DBCONFIG_PASSWORD) - - --enable_database_upgrade=true - command: - - /bin/metadata_store_server - env: - - name: DBCONFIG_USER - valueFrom: - secretKeyRef: - key: username - name: mysql-secret - - name: DBCONFIG_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: mysql-secret - - name: MYSQL_DATABASE - valueFrom: - configMapKeyRef: - key: mlmdDb - name: pipeline-install-config - - name: MYSQL_HOST - valueFrom: - configMapKeyRef: - key: dbHost - name: pipeline-install-config - - name: MYSQL_PORT - valueFrom: - configMapKeyRef: - key: dbPort - name: pipeline-install-config - image: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 - livenessProbe: - initialDelaySeconds: 3 - periodSeconds: 5 - tcpSocket: - port: grpc-api - timeoutSeconds: 2 - name: container - ports: - - containerPort: 8080 - name: grpc-api - readinessProbe: - initialDelaySeconds: 3 - periodSeconds: 5 - tcpSocket: - port: grpc-api - timeoutSeconds: 2 - serviceAccountName: metadata-grpc-server ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: metadata-writer - application-crd-id: kubeflow-pipelines - name: metadata-writer - namespace: kubeflow -spec: - replicas: 1 - selector: - matchLabels: - app: metadata-writer - application-crd-id: kubeflow-pipelines - template: - metadata: - labels: - app: metadata-writer - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: NAMESPACE_TO_WATCH - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: gcr.io/ml-pipeline/metadata-writer:2.0.3 - name: main - serviceAccountName: kubeflow-pipelines-metadata-writer ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: minio - application-crd-id: kubeflow-pipelines - name: minio - namespace: kubeflow -spec: - selector: - matchLabels: - app: minio - application-crd-id: kubeflow-pipelines - strategy: - type: Recreate - template: - metadata: - labels: - app: minio - application-crd-id: kubeflow-pipelines - spec: - containers: - - args: - - server - - /data - env: - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance - name: minio - ports: - - containerPort: 9000 - resources: - requests: - cpu: 20m - memory: 100Mi - volumeMounts: - - mountPath: /data - name: data - subPath: minio - volumes: - - name: data - persistentVolumeClaim: - claimName: minio-pvc ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - name: ml-pipeline - namespace: kubeflow -spec: - selector: - matchLabels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - template: - metadata: - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - labels: - app: ml-pipeline - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION - valueFrom: - configMapKeyRef: - key: autoUpdatePipelineDefaultVersion - name: pipeline-install-config - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: OBJECTSTORECONFIG_SECURE - value: "false" - - name: OBJECTSTORECONFIG_BUCKETNAME - valueFrom: - configMapKeyRef: - key: bucketName - name: pipeline-install-config - - name: DBCONFIG_USER - valueFrom: - secretKeyRef: - key: username - name: mysql-secret - - name: DBCONFIG_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: mysql-secret - - name: DBCONFIG_DBNAME - valueFrom: - configMapKeyRef: - key: pipelineDb - name: pipeline-install-config - - name: DBCONFIG_HOST - valueFrom: - configMapKeyRef: - key: dbHost - name: pipeline-install-config - - name: DBCONFIG_PORT - valueFrom: - configMapKeyRef: - key: dbPort - name: pipeline-install-config - - name: DBCONFIG_CONMAXLIFETIME - valueFrom: - configMapKeyRef: - key: ConMaxLifeTime - name: pipeline-install-config - - name: DB_DRIVER_NAME - valueFrom: - configMapKeyRef: - key: dbType - name: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_USER - valueFrom: - secretKeyRef: - key: username - name: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: mysql-secret - - name: DBCONFIG_MYSQLCONFIG_DBNAME - valueFrom: - configMapKeyRef: - key: pipelineDb - name: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_HOST - valueFrom: - configMapKeyRef: - key: mysqlHost - name: pipeline-install-config - - name: DBCONFIG_MYSQLCONFIG_PORT - valueFrom: - configMapKeyRef: - key: mysqlPort - name: pipeline-install-config - - name: OBJECTSTORECONFIG_ACCESSKEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: OBJECTSTORECONFIG_SECRETACCESSKEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - image: gcr.io/ml-pipeline/api-server:2.0.3 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - name: ml-pipeline-api-server - ports: - - containerPort: 8888 - name: http - - containerPort: 8887 - name: grpc - readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - resources: - requests: - cpu: 250m - memory: 500Mi - startupProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - failureThreshold: 12 - periodSeconds: 5 - timeoutSeconds: 2 - serviceAccountName: ml-pipeline ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: ml-pipeline-persistenceagent - application-crd-id: kubeflow-pipelines - name: ml-pipeline-persistenceagent - namespace: kubeflow -spec: - selector: - matchLabels: - app: ml-pipeline-persistenceagent - application-crd-id: kubeflow-pipelines - template: - metadata: - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - labels: - app: ml-pipeline-persistenceagent - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: TTL_SECONDS_AFTER_WORKFLOW_FINISH - value: "86400" - - name: NUM_WORKERS - value: "2" - image: gcr.io/ml-pipeline/persistenceagent:2.0.3 - imagePullPolicy: IfNotPresent - name: ml-pipeline-persistenceagent - resources: - requests: - cpu: 120m - memory: 500Mi - volumeMounts: - - mountPath: /var/run/secrets/kubeflow/tokens - name: persistenceagent-sa-token - serviceAccountName: ml-pipeline-persistenceagent - volumes: - - name: persistenceagent-sa-token - projected: - sources: - - serviceAccountToken: - audience: pipelines.kubeflow.org - expirationSeconds: 3600 - path: persistenceagent-sa-token ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: ml-pipeline-scheduledworkflow - application-crd-id: kubeflow-pipelines - name: ml-pipeline-scheduledworkflow - namespace: kubeflow -spec: - selector: - matchLabels: - app: ml-pipeline-scheduledworkflow - application-crd-id: kubeflow-pipelines - template: - metadata: - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - labels: - app: ml-pipeline-scheduledworkflow - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: CRON_SCHEDULE_TIMEZONE - valueFrom: - configMapKeyRef: - key: cronScheduleTimezone - name: pipeline-install-config - image: gcr.io/ml-pipeline/scheduledworkflow:2.0.3 - imagePullPolicy: IfNotPresent - name: ml-pipeline-scheduledworkflow - serviceAccountName: ml-pipeline-scheduledworkflow ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines - name: ml-pipeline-ui - namespace: kubeflow -spec: - selector: - matchLabels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines - template: - metadata: - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - labels: - app: ml-pipeline-ui - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH - value: /etc/config/viewer-pod-template.json - - name: MINIO_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MINIO_ACCESS_KEY - valueFrom: - secretKeyRef: - key: accesskey - name: mlpipeline-minio-artifact - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - key: secretkey - name: mlpipeline-minio-artifact - - name: ALLOW_CUSTOM_VISUALIZATIONS - value: "true" - image: gcr.io/ml-pipeline/frontend:2.0.3 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:3000/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - name: ml-pipeline-ui - ports: - - containerPort: 3000 - readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:3000/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - resources: - requests: - cpu: 10m - memory: 70Mi - volumeMounts: - - mountPath: /etc/config - name: config-volume - readOnly: true - serviceAccountName: ml-pipeline-ui - volumes: - - configMap: - name: ml-pipeline-ui-configmap - name: config-volume ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: ml-pipeline-viewer-crd - application-crd-id: kubeflow-pipelines - name: ml-pipeline-viewer-crd - namespace: kubeflow -spec: - selector: - matchLabels: - app: ml-pipeline-viewer-crd - application-crd-id: kubeflow-pipelines - template: - metadata: - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - labels: - app: ml-pipeline-viewer-crd - application-crd-id: kubeflow-pipelines - spec: - containers: - - env: - - name: MAX_NUM_VIEWERS - value: "50" - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: gcr.io/ml-pipeline/viewer-crd-controller:2.0.3 - imagePullPolicy: Always - name: ml-pipeline-viewer-crd - serviceAccountName: ml-pipeline-viewer-crd-service-account ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines - name: ml-pipeline-visualizationserver - namespace: kubeflow -spec: - selector: - matchLabels: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines - template: - metadata: - annotations: - cluster-autoscaler.kubernetes.io/safe-to-evict: "true" - labels: - app: ml-pipeline-visualizationserver - application-crd-id: kubeflow-pipelines - spec: - containers: - - image: gcr.io/ml-pipeline/visualization-server:2.0.3 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/ - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - name: ml-pipeline-visualizationserver - ports: - - containerPort: 8888 - name: http - readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/ - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 - resources: - requests: - cpu: 30m - memory: 500Mi - serviceAccountName: ml-pipeline-visualizationserver ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: mysql - application-crd-id: kubeflow-pipelines - name: mysql - namespace: kubeflow -spec: - selector: - matchLabels: - app: mysql - application-crd-id: kubeflow-pipelines - strategy: - type: Recreate - template: - metadata: - labels: - app: mysql - application-crd-id: kubeflow-pipelines - spec: - containers: - - args: - - --datadir - - /var/lib/mysql - - --default-authentication-plugin=mysql_native_password - - --disable-log-bin - env: - - name: MYSQL_ALLOW_EMPTY_PASSWORD - value: "true" - image: gcr.io/ml-pipeline/mysql:8.0.26 - name: mysql - ports: - - containerPort: 3306 - name: mysql - resources: - requests: - cpu: 100m - memory: 800Mi - volumeMounts: - - mountPath: /var/lib/mysql - name: mysql-persistent-storage - serviceAccountName: mysql - volumes: - - name: mysql-persistent-storage - persistentVolumeClaim: - claimName: mysql-pv-claim ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: proxy-agent - application-crd-id: kubeflow-pipelines - name: proxy-agent - namespace: kubeflow -spec: - selector: - matchLabels: - app: proxy-agent - application-crd-id: kubeflow-pipelines - template: - metadata: - labels: - app: proxy-agent - application-crd-id: kubeflow-pipelines - spec: - containers: - - image: gcr.io/ml-pipeline/inverse-proxy-agent:2.0.3 - imagePullPolicy: IfNotPresent - name: proxy-agent - hostNetwork: true - serviceAccountName: proxy-agent-runner ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apps|Deployment|default|workflow-controller - labels: - application-crd-id: kubeflow-pipelines - name: workflow-controller - namespace: kubeflow -spec: - selector: - matchLabels: - app: workflow-controller - application-crd-id: kubeflow-pipelines - template: - metadata: - labels: - app: workflow-controller - application-crd-id: kubeflow-pipelines - spec: - containers: - - args: - - --configmap - - workflow-controller-configmap - - --executor-image - - gcr.io/ml-pipeline/argoexec:v3.3.10-license-compliance - - --namespaced - command: - - workflow-controller - env: - - name: LEADER_ELECTION_IDENTITY - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - image: gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 6060 - initialDelaySeconds: 90 - periodSeconds: 60 - timeoutSeconds: 30 - name: workflow-controller - ports: - - containerPort: 9090 - name: metrics - - containerPort: 6060 - resources: - requests: - cpu: 100m - memory: 500Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - nodeSelector: - kubernetes.io/os: linux - priorityClassName: workflow-controller - securityContext: - runAsNonRoot: true - serviceAccountName: argo ---- -apiVersion: app.k8s.io/v1beta1 -kind: Application -metadata: - annotations: - kubernetes-engine.cloud.google.com/icon: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADMAAAAyCAYAAADx/eOPAAALuUlEQVRogd2afWxd9XnHP99bK4pS3yhiGUIRiiJUVVVqbq8ppdR20ibqpuIMtDRkUYERp29J57gMZVuxsrZiK7oZXVv5re1AiOuoG+N1DMkuytprsGPEVMouxqQZJVHEWIdQlGVxZlmZdb/747zcc869jpMO+seOdPz7nd/L83tev89zzjX8Bq795Rq9o17zXp+Tey+Ijkyboela29DRWkhffyT733pH/Z3este9F2cC6N0kNjxtjD+FdRD8UF9X7u97y7UbQFPAivC0BdllS381slun3s3z3xVhhqeds90tqR/oMB7u68z19ZZra0E/l1if3WOziPx3skrDPTr+bvDxfxImEIJbgX6gGBJ7EfHJX/ySReDHwO9KYAenyWCMFKw21GSeslwa2Z17+TcuzPBRr7B8m6Df5oOJqdPAR/u6cm/2lmv3At+IT3GiXZqbcaxSLsfRoTsvn7XL2jE87ZXGnwf+VGiDY86ETM1wU1+XjvSW/RlgTJADQ2QaCZKWcX1/aDIcjE8i3SdzZLjn0lm8pJXD02417BM+gLmq2Rqjr/d16Vu95dp6wc8Ra5O8NrPIcoZCvIR1H+KZkd2qLcfnRYUZOuorJO+3uQt0RerolGYZR7r5+C9ZATwPviGyQprd6Liszy3bnwVKwGMjPbnFyxJmeNpX2T4gaR/QmmSpyYZTho/2depMb9k/kNh3KawuJ1bWauHzUcyXRpZAv5Zmg7aHBLcmNN9ECAFeAO3s69KZ3nLtDuF9dnBs0IT9JO24rbPb0JfP2syCZpFfE5q1mRWcvlgMNcwMTRq9z/+OWXdx4AGjvX1deqC37DbwPwOrMgsufol5mWMWs1ivEbjTrOCtLNNb+udygqsNbUBtopR/NkuuwTJ6Hxsw67KSuvH5MPDA/nJttfGTdUFCMUlp/ALwOtIs9muBxpnFnBzuSQf21oP/BbXclVvumWuTaDN8WNBm2GizJkxPM0CDMA2WGZ72bbb/Njue2TRj9Il/PcG87SeBz4ZTNaSTsmctHcO8SqDp14d7dCFLZ2v/3OpQ023Ah4n65kohvETUCdcsfmuilD+bpNdgGZvOODuHqYGIVGCec9g7+7o031v2jaBTiD0ysxbHRnZrPktzyz1zK7f0z10nh5pWwLRhvZro1KqznVJhNB8UyDeSsU4zAOiIXV1OuEqQ2AR79nflXgcY6dGLwIvR8q39cy1b+uc2Emo6dI824BpMSxz8iVhy4m/2WiYHdV5UmOHp2mpwm52ESCdwRn+9v0tPAWzpn9sAFAQbMdc60PaHsFZEWd9uxk4z8G3seykECfObTEd2KmuZG4CWyLXkYLMwtiYt+hMsTUdAEZQzjs9apv66SHJRk73ZjBQ+iRu29s+1VEr5OImmXs4MHUahVoLWgK23wbv6OrU4OulcuHYehWsVHhpXwpE2FNRayTszX2cwDpQEzTB+QvrJHCXUaigk+c++aXZiE98YmUVgV19X7u3ypH/fgfUA5h2usY2jNjmWoGVn50nvC9T2NviA5OPBGPW91OlG+0Xa1WJhhqadk3WjpKCilQIQFP19XZocnfIHgIeFWyNh6goXyX6gdNWfU8aJ5tNjEheAHZVS/ruGj0s8k6VPhh6ms6kwgoLl1aGuCEuSpwXfHZ2qrTJ+HHkNCpOjmbdFcEcGUIhUSj/H65rPO6j+766U8i/QXV0z8cqJc4btwF8AtWgtMb1wj+j41Df/s1EYQwdEDiqM3hDes9quGY3IKoYOvCrU7HlCoZtEWapPkzEpsU8uq8b36a6uBqaBv5l45URLpZT/pmGH8LnkvlAdAOt1oeXqRsuYTjlEMJiXvWN/Z+5szfqioKcOKo7qr/nAEesKiOyv2A/q88rOx8+8bPhK5dUTAA8jbUT6MuKnbKteNVHKP23xCeD1LC0F2TWOmzoAKEiWxmC+sr8rN1OerF2HGaqXFcZhDWaYj11S4ZxcXxVqyKqPZOeNTwM7Jkr5BeDPQJ8NFQaoC/gZ26rXT5TyxxAfRx6P94d0gU0pYYama+tsbwix/AHM4fKUrwAeB68kRJ5AZsWWieGTjLipsVCgrKCwKHF7pZQ/RXf104j76i4ZMmquxkzRXb2zUsqfxdxsfCiA70hRjZbpCDHmJcRdeZPDHkVck0Ul5PeHZ81DgHxKtglXaHCxVN9fr5TyR9hW3QA8Amqp5526SyKtBEbZVv1eZeZkbqKU7xfsFJwPqRW29s+11oUxnUhnkHf2dWoB+R5Jv5dNaGHh1wog8d/ZAI+0GgVpFPTp4AfJT2Hup7u6EvMk0tpkboutEz0HMPzHyD+mu3pFpZR/Aug0Pgm0RLkvFzLWYfjDvs7cqfKUt2LuXTLhue5mdWhVDJdEzxDDcRKawceN9lRePVkDfgBcR/LKVqNpz/s08DO6q4VKKT8j8zHgJ1HyzA1P11YZjfV1arw85auBR4RalDB5lEjDKi0CgPPphKZ0QiNRwUQeg88B2ydKreew9yH1NCxe/r4GaZpt1Vsrh/JnDDcBLwPkbLVgf6s86RXYj4KvtJKJM8KsGLkSlsmUL6mSg1RJY1xD7KmU8sfprnYgBqJsGVsiEfupsca7FfMo26p/OfHKiVqllB8HyPV16VxfV66G/G1QBwY5xvCgTT7X3/MTaBbFVr0fJvqw2ASZ+yul/FN0V68CHsesiDl3UopM3CwhDZDD/Dnwj3S/sjoYAMqTtc1YX02jVqYOiuuqsAKIkqZCfFIz/IrfFY8gDrKt2gI8irSuwQezyTeNaOl+6qYb+fpYGKEXJE9GSTObK5ItrheaLHE5/XRKcHul+kYN8x2kzWlLNNuVtUqibzKW5CBjxUoszO7NWrS1E/xWvMeJjck2WQHEKJeMD+qH4gWCSvg00m3AVxv5TMRKsp9Cs0Q/Ka/1BOZQNBSXMz2b9Q5oO9JCKgkqg2aKofl8uvTPeE1w3t5KKf8y26pFxINhLRa5R9JV6huT/aZuFu7Ds+A9jBdj+VIvZz2b9BL2Xi5yJQEgUFqinI9SZBDx358o5Q/HiRGtquOEmxJu6DcbC/afQWxnvHg+Odrwm2bP5txh5OEYjOM3vaiu8qqHJw1mPmK/Xs7HJf0LRncDMF5cAL6NWUxDrX/duwbczljxjSzvTX+gtXU3MBlrRCltrsxBTgorACKrRGf5bczOiVLrhUL74B2F9oHVjBd/iLwTWEhr+CIWaLYumDjIWLHha+aSwvRs1iJmJ9Kb9ZJRETS3ACsMC8i1ZNwgXZDYWTmU/1WhfeAW8Cjo+UL7wDrGik8jfid0kYz/Z2ODepv+GPIY+FAznpcUJhAo9w5mh81CFtEsWieCTzwXkogmfKBSyh8ttA98EDPqoPouYqYLxYEPMVY8itmEeTM+KEaqZhVAkiPPIL6QDPhLFiYQSC9J7M3mGlF/24zWSvwIM1xoH2gF/sFiTcSPxQakqUJxsIPx4jGCr0AzCUYTbROJ7DPAdsbSAX9ZwgDs3qTDiMGUOxF/1DgfekLVsPf0sw8DPARsDNwy8iYBXov4p0L7wC2MF99CfBJ4rqmbJbO/qYE+x1jx5HK8Xtp/aFgHDM/FX+RM9FFjHjjj4NV3HvlPsP4g+SqQgm6zCuvJQnHgi4wVz2JuAj8RnLGEVaCf8Y8cuRQ2L0mYEBB2Gb8ZHKD4NQBx+0Qpf7LQPrAVVGqiiWTpCcEn4QcLxcF7C7+aXMDahT1YX5IS5DHE/ZfC4yULEwr0DtIOWwuuvwZ8rVLKP1soDqzHPGJoyRao9b4SXiQQ30A8eO1/PJ8D7gK+BtQSJcQM8AXGlg747LUkmi91lad8J3CuZ5OeBii0D64ET2FdH1N0omWJvgLPkvwM8LmZf7lrnm3VO4CHsM4DH2P8I8vGSfK67P9q8v9wWPAcQLH4PbBHbK6Pq+3M9+Ml+6FL2dyC+WmhOLiWseKPMDeDd12uIPBrWCZ5Xds++AHsAwGlBKnoB5747c2J+aSJEuvRL8CDv/2Zz+cqh/LL/gPD//vrfwFjcI5oX6jDBwAAAABJRU5ErkJggg== - marketplace.cloud.google.com/deploy-info: '{"partner_id": "google-cloud-ai-platform", - "product_id": "kubeflow-pipelines", "partner_name": "Google Cloud AI Platform"}' - labels: - application-crd-id: kubeflow-pipelines - name: pipeline - namespace: kubeflow -spec: - addOwnerRef: true - componentKinds: - - group: v1 - kind: ServiceAccount - - group: rbac.authorization.k8s.io/v1 - kind: Role - - group: rbac.authorization.k8s.io/v1 - kind: RoleBinding - - group: v1 - kind: Service - - group: v1 - kind: ConfigMap - - group: v1 - kind: Secret - - group: apps/v1 - kind: Deployment - descriptor: - description: Reusable end-to-end ML workflow - links: - - description: Kubeflow Pipelines Documentation - url: https://www.kubeflow.org/docs/pipelines/ - maintainers: - - name: Google Cloud AI Platform - url: https://cloud.google.com/ai-platform/ - - name: Kubeflow Pipelines - url: https://github.com/kubeflow/pipelines - notes: Please go to [Hosted Kubeflow Pipelines Console](https://console.cloud.google.com/ai-platform/pipelines/clusters). - type: Kubeflow Pipelines - version: 2.0.3 - info: - - name: Console - value: https://console.cloud.google.com/ai-platform/pipelines/clusters - selector: - matchLabels: - application-crd-id: kubeflow-pipelines diff --git a/module-4/kfp-yml/res.yaml b/module-4/kfp-yml/res.yaml deleted file mode 100644 index ca020e6..0000000 --- a/module-4/kfp-yml/res.yaml +++ /dev/null @@ -1,1349 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kubeflow ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - api-approved.kubernetes.io: https://github.com/kubernetes-sigs/application/pull/2 - controller-gen.kubebuilder.io/version: v0.4.0 - creationTimestamp: null - labels: - controller-tools.k8s.io: "1.0" - name: applications.app.k8s.io -spec: - group: app.k8s.io - names: - categories: - - all - kind: Application - listKind: ApplicationList - plural: applications - shortNames: - - app - singular: application - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: The type of the application - jsonPath: .spec.descriptor.type - name: Type - type: string - - description: The creation date - jsonPath: .spec.descriptor.version - name: Version - type: string - - description: The application object owns the matched resources - jsonPath: .spec.addOwnerRef - name: Owner - type: boolean - - description: Numbers of components ready - jsonPath: .status.componentsReady - name: Ready - type: string - - description: The creation date - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1beta1 - schema: - openAPIV3Schema: - description: Application is the Schema for the applications API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ApplicationSpec defines the specification for an Application. - properties: - addOwnerRef: - description: AddOwnerRef objects - flag to indicate if we need to - add OwnerRefs to matching objects Matching is done by using Selector - to query all ComponentGroupKinds - type: boolean - assemblyPhase: - description: AssemblyPhase represents the current phase of the application's - assembly. An empty value is equivalent to "Succeeded". - type: string - componentKinds: - description: ComponentGroupKinds is a list of Kinds for Application's - components (e.g. Deployments, Pods, Services, CRDs). It can be used - in conjunction with the Application's Selector to list or watch - the Applications components. - items: - description: GroupKind specifies a Group and a Kind, but does not - force a version. This is useful for identifying concepts during - lookup stages without having partially valid types - properties: - group: - type: string - kind: - type: string - required: - - group - - kind - type: object - type: array - descriptor: - description: Descriptor regroups information and metadata about an - application. - properties: - description: - description: Description is a brief string description of the - Application. - type: string - icons: - description: Icons is an optional list of icons for an application. - Icon information includes the source, size, and mime type. - items: - description: ImageSpec contains information about an image used - as an icon. - properties: - size: - description: (optional) The size of the image in pixels - (e.g., 25x25). - type: string - src: - description: The source for image represented as either - an absolute URL to the image or a Data URL containing - the image. Data URLs are defined in RFC 2397. - type: string - type: - description: (optional) The mine type of the image (e.g., - "image/png"). - type: string - required: - - src - type: object - type: array - keywords: - description: Keywords is an optional list of key words associated - with the application (e.g. MySQL, RDBMS, database). - items: - type: string - type: array - links: - description: Links are a list of descriptive URLs intended to - be used to surface additional documentation, dashboards, etc. - items: - description: Link contains information about an URL to surface - documentation, dashboards, etc. - properties: - description: - description: Description is human readable content explaining - the purpose of the link. - type: string - url: - description: Url typically points at a website address. - type: string - type: object - type: array - maintainers: - description: Maintainers is an optional list of maintainers of - the application. The maintainers in this list maintain the the - source code, images, and package for the application. - items: - description: ContactData contains information about an individual - or organization. - properties: - email: - description: Email is the email address. - type: string - name: - description: Name is the descriptive name. - type: string - url: - description: Url could typically be a website address. - type: string - type: object - type: array - notes: - description: Notes contain a human readable snippets intended - as a quick start for the users of the Application. CommonMark - markdown syntax may be used for rich text representation. - type: string - owners: - description: Owners is an optional list of the owners of the installed - application. The owners of the application should be contacted - in the event of a planned or unplanned disruption affecting - the application. - items: - description: ContactData contains information about an individual - or organization. - properties: - email: - description: Email is the email address. - type: string - name: - description: Name is the descriptive name. - type: string - url: - description: Url could typically be a website address. - type: string - type: object - type: array - type: - description: Type is the type of the application (e.g. WordPress, - MySQL, Cassandra). - type: string - version: - description: Version is an optional version indicator for the - Application. - type: string - type: object - info: - description: Info contains human readable key,value pairs for the - Application. - items: - description: InfoItem is a human readable key,value pair containing - important information about how to access the Application. - properties: - name: - description: Name is a human readable title for this piece of - information. - type: string - type: - description: Type of the value for this InfoItem. - type: string - value: - description: Value is human readable content. - type: string - valueFrom: - description: ValueFrom defines a reference to derive the value - from another source. - properties: - configMapKeyRef: - description: Selects a key of a ConfigMap. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead - of an entire object, this string should contain a - valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container - within a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container - that triggered the event) or if no container name - is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to - have some well-defined way of referencing a part of - an object. TODO: this design is not final and this - field is subject to change in the future.' - type: string - key: - description: The key to select. - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this - reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - ingressRef: - description: Select an Ingress. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead - of an entire object, this string should contain a - valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container - within a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container - that triggered the event) or if no container name - is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to - have some well-defined way of referencing a part of - an object. TODO: this design is not final and this - field is subject to change in the future.' - type: string - host: - description: The optional host to select. - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - path: - description: The optional HTTP path. - type: string - protocol: - description: Protocol for the ingress - type: string - resourceVersion: - description: 'Specific resourceVersion to which this - reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - secretKeyRef: - description: Selects a key of a Secret. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead - of an entire object, this string should contain a - valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container - within a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container - that triggered the event) or if no container name - is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to - have some well-defined way of referencing a part of - an object. TODO: this design is not final and this - field is subject to change in the future.' - type: string - key: - description: The key to select. - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this - reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - serviceRef: - description: Select a Service. - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead - of an entire object, this string should contain a - valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container - within a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container - that triggered the event) or if no container name - is specified "spec.containers[2]" (container with - index 2 in this pod). This syntax is chosen only to - have some well-defined way of referencing a part of - an object. TODO: this design is not final and this - field is subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: - https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - path: - description: The optional HTTP path. - type: string - port: - description: The optional port to select. - format: int32 - type: integer - protocol: - description: Protocol for the service - type: string - resourceVersion: - description: 'Specific resourceVersion to which this - reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - type: - description: Type of source. - type: string - type: object - type: object - type: array - selector: - description: 'Selector is a label query over kinds that created by - the application. It must match the component objects'' labels. More - info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. - The requirements are ANDed. - items: - description: A label selector requirement is a selector that - contains values, a key, and an operator that relates the key - and values. - properties: - key: - description: key is the label key that the selector applies - to. - type: string - operator: - description: operator represents a key's relationship to - a set of values. Valid operators are In, NotIn, Exists - and DoesNotExist. - type: string - values: - description: values is an array of string values. If the - operator is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single - {key,value} in the matchLabels map is equivalent to an element - of matchExpressions, whose key field is "key", the operator - is "In", and the values array contains only "value". The requirements - are ANDed. - type: object - type: object - type: object - status: - description: ApplicationStatus defines controller's the observed state - of Application - properties: - components: - description: Object status array for all matching objects - items: - description: ObjectStatus is a generic status holder for objects - properties: - group: - description: Object group - type: string - kind: - description: Kind of object - type: string - link: - description: Link to object - type: string - name: - description: Name of object - type: string - status: - description: 'Status. Values: InProgress, Ready, Unknown' - type: string - type: object - type: array - componentsReady: - description: 'ComponentsReady: status of the components in the format - ready/total' - type: string - conditions: - description: Conditions represents the latest state of the object - items: - description: Condition describes the state of an object at a certain - point. - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status - to another. - format: date-time - type: string - lastUpdateTime: - description: Last time the condition was probed - format: date-time - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition. - type: string - required: - - status - - type - type: object - type: array - observedGeneration: - description: ObservedGeneration is the most recent generation observed. - It corresponds to the Object's generation, which is updated on mutation - by the API Server. - format: int64 - type: integer - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apiextensions.k8s.io|CustomResourceDefinition|default|clusterworkflowtemplates.argoproj.io - name: clusterworkflowtemplates.argoproj.io -spec: - group: argoproj.io - names: - kind: ClusterWorkflowTemplate - listKind: ClusterWorkflowTemplateList - plural: clusterworkflowtemplates - shortNames: - - clusterwftmpl - - cwft - singular: clusterworkflowtemplate - scope: Cluster - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - metadata - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apiextensions.k8s.io|CustomResourceDefinition|default|cronworkflows.argoproj.io - name: cronworkflows.argoproj.io -spec: - group: argoproj.io - names: - kind: CronWorkflow - listKind: CronWorkflowList - plural: cronworkflows - shortNames: - - cwf - - cronwf - singular: cronworkflow - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - status: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - metadata - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: scheduledworkflows.kubeflow.org -spec: - group: kubeflow.org - names: - kind: ScheduledWorkflow - listKind: ScheduledWorkflowList - plural: scheduledworkflows - shortNames: - - swf - singular: scheduledworkflow - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - status: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - spec - - status - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: viewers.kubeflow.org -spec: - group: kubeflow.org - names: - kind: Viewer - listKind: ViewerList - plural: viewers - shortNames: - - vi - singular: viewer - scope: Namespaced - versions: - - name: v1beta1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apiextensions.k8s.io|CustomResourceDefinition|default|workfloweventbindings.argoproj.io - name: workfloweventbindings.argoproj.io -spec: - group: argoproj.io - names: - kind: WorkflowEventBinding - listKind: WorkflowEventBindingList - plural: workfloweventbindings - shortNames: - - wfeb - singular: workfloweventbinding - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - metadata - - spec - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apiextensions.k8s.io|CustomResourceDefinition|default|workflows.argoproj.io - name: workflows.argoproj.io -spec: - group: argoproj.io - names: - kind: Workflow - listKind: WorkflowList - plural: workflows - shortNames: - - wf - singular: workflow - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Status of the workflow - jsonPath: .status.phase - name: Status - type: string - - description: When the workflow was started - format: date-time - jsonPath: .status.startedAt - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - status: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - metadata - - spec - type: object - served: true - storage: true - subresources: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apiextensions.k8s.io|CustomResourceDefinition|default|workflowtaskresults.argoproj.io - name: workflowtaskresults.argoproj.io -spec: - group: argoproj.io - names: - kind: WorkflowTaskResult - listKind: WorkflowTaskResultList - plural: workflowtaskresults - singular: workflowtaskresult - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - message: - type: string - metadata: - type: object - outputs: - properties: - artifacts: - items: - properties: - archive: - properties: - none: - type: object - tar: - properties: - compressionLevel: - format: int32 - type: integer - type: object - zip: - type: object - type: object - archiveLogs: - type: boolean - artifactory: - properties: - passwordSecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - url: - type: string - usernameSecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - required: - - url - type: object - from: - type: string - fromExpression: - type: string - gcs: - properties: - bucket: - type: string - key: - type: string - serviceAccountKeySecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - required: - - key - type: object - git: - properties: - depth: - format: int64 - type: integer - disableSubmodules: - type: boolean - fetch: - items: - type: string - type: array - insecureIgnoreHostKey: - type: boolean - passwordSecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - repo: - type: string - revision: - type: string - sshPrivateKeySecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - usernameSecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - required: - - repo - type: object - globalName: - type: string - hdfs: - properties: - addresses: - items: - type: string - type: array - force: - type: boolean - hdfsUser: - type: string - krbCCacheSecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - krbConfigConfigMap: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - krbKeytabSecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - krbRealm: - type: string - krbServicePrincipalName: - type: string - krbUsername: - type: string - path: - type: string - required: - - path - type: object - http: - properties: - headers: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - url: - type: string - required: - - url - type: object - mode: - format: int32 - type: integer - name: - type: string - optional: - type: boolean - oss: - properties: - accessKeySecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - bucket: - type: string - createBucketIfNotPresent: - type: boolean - endpoint: - type: string - key: - type: string - lifecycleRule: - properties: - markDeletionAfterDays: - format: int32 - type: integer - markInfrequentAccessAfterDays: - format: int32 - type: integer - type: object - secretKeySecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - securityToken: - type: string - required: - - key - type: object - path: - type: string - raw: - properties: - data: - type: string - required: - - data - type: object - recurseMode: - type: boolean - s3: - properties: - accessKeySecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - bucket: - type: string - createBucketIfNotPresent: - properties: - objectLocking: - type: boolean - type: object - encryptionOptions: - properties: - enableEncryption: - type: boolean - kmsEncryptionContext: - type: string - kmsKeyId: - type: string - serverSideCustomerKeySecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - endpoint: - type: string - insecure: - type: boolean - key: - type: string - region: - type: string - roleARN: - type: string - secretKeySecret: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - useSDKCreds: - type: boolean - type: object - subPath: - type: string - required: - - name - type: object - type: array - exitCode: - type: string - parameters: - items: - properties: - default: - type: string - description: - type: string - enum: - items: - type: string - type: array - globalName: - type: string - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - default: - type: string - event: - type: string - expression: - type: string - jqFilter: - type: string - jsonPath: - type: string - parameter: - type: string - path: - type: string - supplied: - type: object - type: object - required: - - name - type: object - type: array - result: - type: string - type: object - phase: - type: string - progress: - type: string - required: - - metadata - type: object - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apiextensions.k8s.io|CustomResourceDefinition|default|workflowtasksets.argoproj.io - name: workflowtasksets.argoproj.io -spec: - group: argoproj.io - names: - kind: WorkflowTaskSet - listKind: WorkflowTaskSetList - plural: workflowtasksets - shortNames: - - wfts - singular: workflowtaskset - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - status: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - metadata - - spec - type: object - served: true - storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - internal.kpt.dev/upstream-identifier: apiextensions.k8s.io|CustomResourceDefinition|default|workflowtemplates.argoproj.io - name: workflowtemplates.argoproj.io -spec: - group: argoproj.io - names: - kind: WorkflowTemplate - listKind: WorkflowTemplateList - plural: workflowtemplates - shortNames: - - wftmpl - singular: workflowtemplate - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - type: object - x-kubernetes-map-type: atomic - x-kubernetes-preserve-unknown-fields: true - required: - - metadata - - spec - type: object - served: true - storage: true ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubeflow-pipelines-cache-deployer-sa - namespace: kubeflow ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app: kubeflow-pipelines-cache-deployer-clusterrole - name: kubeflow-pipelines-cache-deployer-clusterrole -rules: -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - - certificatesigningrequests/approval - verbs: - - create - - delete - - get - - update -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - verbs: - - create - - delete - - get - - list - - patch -- apiGroups: - - certificates.k8s.io - resourceNames: - - kubernetes.io/* - resources: - - signers - verbs: - - approve ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubeflow-pipelines-cache-deployer-clusterrolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubeflow-pipelines-cache-deployer-clusterrole -subjects: -- kind: ServiceAccount - name: kubeflow-pipelines-cache-deployer-sa - namespace: kubeflow diff --git a/module-4/kubeflow_pipelines/kfp-inference-pipeline_v2.py b/module-4/kubeflow_pipelines/kfp_inference_pipeline.py similarity index 79% rename from module-4/kubeflow_pipelines/kfp-inference-pipeline_v2.py rename to module-4/kubeflow_pipelines/kfp_inference_pipeline.py index aa81299..0e83383 100644 --- a/module-4/kubeflow_pipelines/kfp-inference-pipeline_v2.py +++ b/module-4/kubeflow_pipelines/kfp_inference_pipeline.py @@ -1,29 +1,25 @@ import os import uuid from typing import Optional -from kfp.client import Client + import kfp import typer from kfp import dsl -from kubernetes.client.models import V1EnvVar -from kfp import dsl -from kfp import compiler -from kfp.dsl import Input, Output, Dataset, Model, Artifact -from typing import NamedTuple +from kfp.dsl import Artifact, Dataset, Input, Model, Output - -IMAGE = "kyrylprojector/nlp-sample:latest" +IMAGE = "ghcr.io/kyryl-opens-ml/classic-example:main" WANDB_PROJECT = os.getenv("WANDB_PROJECT") WANDB_API_KEY = os.getenv("WANDB_API_KEY") @dsl.component(base_image=IMAGE) def load_data(test_data: Output[Dataset]): - from nlp_sample.data import load_cola_data - from pathlib import Path import shutil + from pathlib import Path + + from classic_example.data import load_sst2_data - load_cola_data(Path("/app/data")) + load_sst2_data(Path("/app/data")) shutil.move(Path("/app/data") / "test.csv", test_data.path) @@ -37,9 +33,10 @@ def load_model( model_card: Output[Artifact], special_tokens_map: Output[Artifact], ): - from nlp_sample.utils import load_from_registry - from pathlib import Path import shutil + from pathlib import Path + + from classic_example.utils import load_from_registry model_path = Path("/tmp/model") model_path.mkdir(exist_ok=True) @@ -64,9 +61,10 @@ def run_inference( test_data: Input[Dataset], pred: Output[Dataset], ): - from nlp_sample.predictor import run_inference_on_dataframe - from pathlib import Path import shutil + from pathlib import Path + + from classic_example.predictor import run_inference_on_dataframe model_path = Path("/tmp/model") model_path.mkdir(exist_ok=True) @@ -77,7 +75,9 @@ def run_inference( shutil.copy(special_tokens_map.path, model_path / "special_tokens_map.json") shutil.copy(model_card.path, model_path / "README.md") - run_inference_on_dataframe(df_path=test_data.path, model_load_path=model_path, result_path=pred.path) + run_inference_on_dataframe( + df_path=test_data.path, model_load_path=model_path, result_path=pred.path + ) @dsl.pipeline @@ -85,8 +85,12 @@ def inference_pipeline(): load_data_task = load_data() load_model_task = load_model() - load_model_task = load_model_task.set_env_variable(name="WANDB_PROJECT", value=WANDB_PROJECT) - load_model_task = load_model_task.set_env_variable(name="WANDB_API_KEY", value=WANDB_API_KEY) + load_model_task = load_model_task.set_env_variable( + name="WANDB_PROJECT", value=WANDB_PROJECT + ) + load_model_task = load_model_task.set_env_variable( + name="WANDB_API_KEY", value=WANDB_API_KEY + ) run_inference_task = run_inference( config=load_model_task.outputs["config"], @@ -100,7 +104,7 @@ def inference_pipeline(): def compile_pipeline() -> str: - path = "/tmp/nlp_inference_pipeline.yaml" + path = "/tmp/inference_pipeline.yaml" kfp.compiler.Compiler().compile(inference_pipeline, path) return path @@ -110,7 +114,7 @@ def create_pipeline(client: kfp.Client, namespace: str): _ = client.create_experiment("inference", namespace=namespace) print("Uploading pipeline") - name = "nlp-sample-inference" + name = "classic-example-inference" if client.get_pipeline_id(name) is not None: print("Pipeline exists - upload new version.") pipeline_prev_version = client.get_pipeline(client.get_pipeline_id(name)) @@ -121,7 +125,9 @@ def create_pipeline(client: kfp.Client, namespace: str): pipeline_id=pipeline_prev_version.pipeline_id, ) else: - pipeline = client.upload_pipeline(pipeline_package_path=compile_pipeline(), pipeline_name=name) + pipeline = client.upload_pipeline( + pipeline_package_path=compile_pipeline(), pipeline_name=name + ) print(f"pipeline {pipeline.pipeline_id}") diff --git a/module-4/kubeflow_pipelines/kfp-training-pipeline_v2.py b/module-4/kubeflow_pipelines/kfp_training_pipeline.py similarity index 78% rename from module-4/kubeflow_pipelines/kfp-training-pipeline_v2.py rename to module-4/kubeflow_pipelines/kfp_training_pipeline.py index c74385d..b4c0e40 100644 --- a/module-4/kubeflow_pipelines/kfp-training-pipeline_v2.py +++ b/module-4/kubeflow_pipelines/kfp_training_pipeline.py @@ -1,29 +1,27 @@ import os import uuid from typing import Optional -from kfp.client import Client + import kfp import typer from kfp import dsl -from kubernetes.client.models import V1EnvVar -from kfp import dsl -from kfp import compiler -from kfp.dsl import Input, Output, Dataset, Model, Artifact -from typing import NamedTuple - +from kfp.dsl import Artifact, Dataset, Input, Model, Output -IMAGE = "kyrylprojector/nlp-sample:latest" +IMAGE = "ghcr.io/kyryl-opens-ml/classic-example:main" WANDB_PROJECT = os.getenv("WANDB_PROJECT") WANDB_API_KEY = os.getenv("WANDB_API_KEY") @dsl.component(base_image=IMAGE) -def load_data(train_data: Output[Dataset], val_data: Output[Dataset], test_data: Output[Dataset]): - from nlp_sample.data import load_cola_data - from pathlib import Path +def load_data( + train_data: Output[Dataset], val_data: Output[Dataset], test_data: Output[Dataset] +): import shutil + from pathlib import Path + + from classic_example.data import load_sst2_data - load_cola_data(Path("/app/data")) + load_sst2_data(Path("/app/data")) shutil.move(Path("/app/data") / "train.csv", train_data.path) shutil.move(Path("/app/data") / "val.csv", val_data.path) @@ -42,9 +40,10 @@ def train_model( val_data: Input[Dataset], test_data: Input[Dataset], ): - from nlp_sample.train import train - from pathlib import Path import shutil + from pathlib import Path + + from classic_example.train import train Path("/tmp/data").mkdir(exist_ok=True) shutil.copy(train_data.path, Path("/tmp/data") / "train.csv") @@ -70,9 +69,10 @@ def upload_model( model_card: Input[Artifact], special_tokens_map: Input[Artifact], ): - from nlp_sample.utils import upload_to_registry - from pathlib import Path import shutil + from pathlib import Path + + from classic_example.utils import upload_to_registry model_path = Path("/tmp/model") model_path.mkdir(exist_ok=True) @@ -95,8 +95,12 @@ def training_pipeline(): val_data=load_data_task.outputs["val_data"], test_data=load_data_task.outputs["test_data"], ) - train_model_task = train_model_task.set_env_variable(name="WANDB_PROJECT", value=WANDB_PROJECT) - train_model_task = train_model_task.set_env_variable(name="WANDB_API_KEY", value=WANDB_API_KEY) + train_model_task = train_model_task.set_env_variable( + name="WANDB_PROJECT", value=WANDB_PROJECT + ) + train_model_task = train_model_task.set_env_variable( + name="WANDB_API_KEY", value=WANDB_API_KEY + ) upload_model_task = upload_model( config=train_model_task.outputs["config"], @@ -106,12 +110,16 @@ def training_pipeline(): model_card=train_model_task.outputs["model_card"], special_tokens_map=train_model_task.outputs["special_tokens_map"], ) - upload_model_task = upload_model_task.set_env_variable(name="WANDB_PROJECT", value=WANDB_PROJECT) - upload_model_task = upload_model_task.set_env_variable(name="WANDB_API_KEY", value=WANDB_API_KEY) + upload_model_task = upload_model_task.set_env_variable( + name="WANDB_PROJECT", value=WANDB_PROJECT + ) + upload_model_task = upload_model_task.set_env_variable( + name="WANDB_API_KEY", value=WANDB_API_KEY + ) def compile_pipeline() -> str: - path = "/tmp/nlp_traininig_pipeline.yaml" + path = "/tmp/traininig_pipeline.yaml" kfp.compiler.Compiler().compile(training_pipeline, path) return path @@ -121,7 +129,7 @@ def create_pipeline(client: kfp.Client, namespace: str): _ = client.create_experiment("training", namespace=namespace) print("Uploading pipeline") - name = "nlp-sample-training" + name = "classic-example-training" if client.get_pipeline_id(name) is not None: print("Pipeline exists - upload new version.") pipeline_prev_version = client.get_pipeline(client.get_pipeline_id(name)) @@ -132,7 +140,9 @@ def create_pipeline(client: kfp.Client, namespace: str): pipeline_id=pipeline_prev_version.pipeline_id, ) else: - pipeline = client.upload_pipeline(pipeline_package_path=compile_pipeline(), pipeline_name=name) + pipeline = client.upload_pipeline( + pipeline_package_path=compile_pipeline(), pipeline_name=name + ) print(f"pipeline {pipeline.pipeline_id}") diff --git a/module-4/requirements.txt b/module-4/requirements.txt index 8d20dda..bdb2b68 100644 --- a/module-4/requirements.txt +++ b/module-4/requirements.txt @@ -1,4 +1,3 @@ -kfp==1.8.9 +kfp==2.8.0 apache-airflow==2.9.2 -apache-airflow-providers-cncf-kubernetes==1.0.0 -cached-property==1.5.2 \ No newline at end of file +apache-airflow-providers-cncf-kubernetes==8.3.3