kubeflow.yaml

gistfile1.txt

"apiVersion": |-
  argoproj.io/v1alpha1
"kind": |-
  Workflow
"metadata":
  "annotations":
    "pipelines.kubeflow.org/pipeline_spec": |-
      {"description": "A pipeline with two sequential steps.", "inputs": [{"default": "gs://kubeflow-ryanbouamara/kc_house_data.csv", "name": "url", "optional": true}, {"default": "/mnt/trained_model", "name": "trained_path", "optional": true}, {"default": "/mnt/dataset.csv", "name": "data_path", "optional": true}], "name": "Sequential pipeline"}
  "generateName": |-
    sequential-pipeline-
"spec":
  "arguments":
    "parameters":
    - "name": |-
        url
      "value": |-
        gs://kubeflow-ryanbouamara/kc_house_data.csv
    - "name": |-
        trained_path
      "value": |-
        /mnt/trained_model
    - "name": |-
        data_path
      "value": |-
        /mnt/dataset.csv
  "entrypoint": |-
    sequential-pipeline
  "serviceAccountName": |-
    pipeline-runner
  "templates":
  - "name": |-
      creation-of-volume
    "outputs":
      "parameters":
      - "name": |-
          creation-of-volume-manifest
        "valueFrom":
          "jsonPath": |-
            {}
      - "name": |-
          creation-of-volume-name
        "valueFrom":
          "jsonPath": |-
            {.metadata.name}
      - "name": |-
          creation-of-volume-size
        "valueFrom":
          "jsonPath": |-
            {.status.capacity.storage}
    "resource":
      "action": |-
        create
      "manifest": |
        apiVersion: v1
        kind: PersistentVolumeClaim
        metadata:
          name: '{{workflow.name}}-dataset-pvc'
        spec:
          accessModes:
          - ReadWriteOnce
          resources:
            requests:
              storage: 1Gi
  - "container":
      "args":
      - |-
        gsutil cat $0 | tee $1
      - |-
        {{inputs.parameters.url}}
      - |-
        {{inputs.parameters.data_path}}
      "command":
      - |-
        sh
      - |-
        -c
      "image": |-
        google/cloud-sdk:272.0.0
      "resources":
        "limits":
          "cpu": |-
            1
          "memory": |-
            2G
        "requests":
          "cpu": |-
            0.5
          "memory": |-
            1G
      "volumeMounts":
      - "mountPath": |-
          /mnt
        "name": |-
          creation-of-volume
    "inputs":
      "parameters":
      - "name": |-
          creation-of-volume-name
      - "name": |-
          data_path
      - "name": |-
          url
    "name": |-
      download-dataset
    "volumes":
    - "name": |-
        creation-of-volume
      "persistentVolumeClaim":
        "claimName": |-
          {{inputs.parameters.creation-of-volume-name}}
  - "container":
      "args":
      - |-
        python3 preprocess.py
      "command":
      - |-
        sh
      - |-
        -c
      "env":
      - "name": |-
          TRAINED_MODEL_PATH
        "value": |-
          {{inputs.parameters.trained_path}}
      - "name": |-
          DATA_PATH
        "value": |-
          {{inputs.parameters.data_path}}
      "image": |-
        anyr/workshop:python2
      "resources":
        "limits":
          "cpu": |-
            1
          "memory": |-
            4G
        "requests":
          "cpu": |-
            1
          "memory": |-
            2G
      "volumeMounts":
      - "mountPath": |-
          /mnt
        "name": |-
          creation-of-volume
    "inputs":
      "parameters":
      - "name": |-
          creation-of-volume-name
      - "name": |-
          data_path
      - "name": |-
          trained_path
    "name": |-
      preprocess-step
    "volumes":
    - "name": |-
        creation-of-volume
      "persistentVolumeClaim":
        "claimName": |-
          {{inputs.parameters.creation-of-volume-name}}
  - "dag":
      "tasks":
      - "name": |-
          creation-of-volume
        "template": |-
          creation-of-volume
      - "arguments":
          "parameters":
          - "name": |-
              creation-of-volume-name
            "value": |-
              {{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
          - "name": |-
              data_path
            "value": |-
              {{inputs.parameters.data_path}}
          - "name": |-
              url
            "value": |-
              {{inputs.parameters.url}}
        "dependencies":
        - |-
          creation-of-volume
        "name": |-
          download-dataset
        "template": |-
          download-dataset
      - "arguments":
          "parameters":
          - "name": |-
              creation-of-volume-name
            "value": |-
              {{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
          - "name": |-
              data_path
            "value": |-
              {{inputs.parameters.data_path}}
          - "name": |-
              trained_path
            "value": |-
              {{inputs.parameters.trained_path}}
        "dependencies":
        - |-
          creation-of-volume
        - |-
          download-dataset
        "name": |-
          preprocess-step
        "template": |-
          preprocess-step
      - "arguments":
          "parameters":
          - "name": |-
              creation-of-volume-name
            "value": |-
              {{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
          - "name": |-
              data_path
            "value": |-
              {{inputs.parameters.data_path}}
          - "name": |-
              trained_path
            "value": |-
              {{inputs.parameters.trained_path}}
        "dependencies":
        - |-
          creation-of-volume
        - |-
          preprocess-step
        "name": |-
          train-step
        "template": |-
          train-step
      - "arguments":
          "parameters":
          - "name": |-
              creation-of-volume-name
            "value": |-
              {{tasks.creation-of-volume.outputs.parameters.creation-of-volume-name}}
          - "name": |-
              data_path
            "value": |-
              {{inputs.parameters.data_path}}
          - "name": |-
              trained_path
            "value": |-
              {{inputs.parameters.trained_path}}
        "dependencies":
        - |-
          creation-of-volume
        - |-
          train-step
        "name": |-
          val-step
        "template": |-
          val-step
    "inputs":
      "parameters":
      - "name": |-
          data_path
      - "name": |-
          trained_path
      - "name": |-
          url
    "name": |-
      sequential-pipeline
  - "container":
      "args":
      - |-
        python3 train.py
      "command":
      - |-
        sh
      - |-
        -c
      "env":
      - "name": |-
          TRAINED_MODEL_PATH
        "value": |-
          {{inputs.parameters.trained_path}}
      - "name": |-
          DATA_PATH
        "value": |-
          {{inputs.parameters.data_path}}
      "image": |-
        anyr/workshop:python2
      "resources":
        "limits":
          "cpu": |-
            10
          "memory": |-
            4G
        "requests":
          "cpu": |-
            5
          "memory": |-
            2G
      "volumeMounts":
      - "mountPath": |-
          /mnt
        "name": |-
          creation-of-volume
    "inputs":
      "parameters":
      - "name": |-
          creation-of-volume-name
      - "name": |-
          data_path
      - "name": |-
          trained_path
    "name": |-
      train-step
    "volumes":
    - "name": |-
        creation-of-volume
      "persistentVolumeClaim":
        "claimName": |-
          {{inputs.parameters.creation-of-volume-name}}
  - "container":
      "args":
      - |-
        python3 val.py; ls /
      "command":
      - |-
        sh
      - |-
        -c
      "env":
      - "name": |-
          TRAINED_MODEL_PATH
        "value": |-
          {{inputs.parameters.trained_path}}
      - "name": |-
          DATA_PATH
        "value": |-
          {{inputs.parameters.data_path}}
      "image": |-
        anyr/workshop:python2
      "resources":
        "limits":
          "cpu": |-
            1
          "memory": |-
            4G
        "requests":
          "cpu": |-
            1
          "memory": |-
            2G
      "volumeMounts":
      - "mountPath": |-
          /mnt
        "name": |-
          creation-of-volume
    "inputs":
      "parameters":
      - "name": |-
          creation-of-volume-name
      - "name": |-
          data_path
      - "name": |-
          trained_path
    "name": |-
      val-step
    "volumes":
    - "name": |-
        creation-of-volume
      "persistentVolumeClaim":
        "claimName": |-
          {{inputs.parameters.creation-of-volume-name}}