Skip to content

Instantly share code, notes, and snippets.

@damitkwr

damitkwr/logs.json

Created August 29, 2019 17:41
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save damitkwr/a25c9bc0ca5361ccddd662a427fe5ee9 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save damitkwr/a25c9bc0ca5361ccddd662a427fe5ee9 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
logs.json
[{
"textPayload": "2019/08/28 18:18:34 http: TLS handshake error from 10.24.0.29:42900: tls: first record does not look like a TLS handshake\n",
"insertId": "1no3685g4nov8mh",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:18:34.715535846Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:18:40.974313368Z"
},
{
"textPayload": "2019/08/28 18:18:19 http: TLS handshake error from 10.24.0.29:42700: tls: first record does not look like a TLS handshake\n",
"insertId": "o538f2g4ntjtr5",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:18:19.715480278Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:18:25.883791995Z"
},
{
"textPayload": "2019/08/28 18:18:04 http: TLS handshake error from 10.24.0.29:42506: tls: first record does not look like a TLS handshake\n",
"insertId": "zsocn7f7uf20q",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:18:04.715766602Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:18:10.701125957Z"
},
{
"textPayload": "2019/08/28 18:17:49 http: TLS handshake error from 10.24.0.29:42320: tls: first record does not look like a TLS handshake\n",
"insertId": "12amstcg3r6gneg",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:17:49.715543584Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:17:55.847244053Z"
},
{
"textPayload": "2019/08/28 18:17:34 http: TLS handshake error from 10.24.0.29:42130: tls: first record does not look like a TLS handshake\n",
"insertId": "7rx041f72bh79",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:17:34.715528648Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:17:40.770212789Z"
},
{
"textPayload": "2019/08/28 18:17:19 http: TLS handshake error from 10.24.0.29:41930: tls: first record does not look like a TLS handshake\n",
"insertId": "bdn4vkfezbun7",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:17:19.715437809Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:17:24.555867151Z"
},
{
"textPayload": "2019/08/28 18:17:04 http: TLS handshake error from 10.24.0.29:41732: tls: first record does not look like a TLS handshake\n",
"insertId": "dbh2fcg5kk8azw",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:17:04.715375050Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:17:11.249446036Z"
},
{
"textPayload": "2019/08/28 18:16:49 http: TLS handshake error from 10.24.0.29:41542: tls: first record does not look like a TLS handshake\n",
"insertId": "vd5r0sf7bieek",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:16:49.715248506Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:16:56.230529652Z"
},
{
"textPayload": "2019/08/28 18:16:34 http: TLS handshake error from 10.24.0.29:41352: tls: first record does not look like a TLS handshake\n",
"insertId": "u96ewg4pe59yy",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:16:34.715596494Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:16:41.163402088Z"
},
{
"textPayload": "2019/08/28 18:16:19 http: TLS handshake error from 10.24.0.29:41152: tls: first record does not look like a TLS handshake\n",
"insertId": "1fm57jfg2lynckk",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:16:19.715457693Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:16:26.085198886Z"
},
{
"textPayload": "2019/08/28 18:16:04 http: TLS handshake error from 10.24.0.29:40958: tls: first record does not look like a TLS handshake\n",
"insertId": "lnhz64g4po6eeh",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:16:04.715252016Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:16:11.025065404Z"
},
{
"textPayload": "2019/08/28 18:15:49 http: TLS handshake error from 10.24.0.29:40770: tls: first record does not look like a TLS handshake\n",
"insertId": "bnnppgg2q3dxlc",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:49.715476053Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:55.971147593Z"
},
{
"textPayload": "2019/08/28 18:15:34 http: TLS handshake error from 10.24.0.29:40580: tls: first record does not look like a TLS handshake\n",
"insertId": "2idy9xfeghv12",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:15:34.715726060Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:40.907653840Z"
},
{
"textPayload": "2019/08/28 18:15:19 http: TLS handshake error from 10.24.0.29:40380: tls: first record does not look like a TLS handshake\n",
"insertId": "o538f2g4nsvy7s",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:15:19.715251600Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:21.738624603Z"
},
{
"insertId": "1qzzb5zg4niwa22",
"jsonPayload": {
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567016109.9818137,
"msg": "Found identical Istio Destination Rule",
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.981997522Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa21",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc",
"namespace": "seldon",
"ts": 1567016109.9817939,
"msg": "Found identical Virtual Service"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:15:09.981993137Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa20",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http",
"namespace": "seldon",
"ts": 1567016109.9817617,
"msg": "Found identical Virtual Service"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.981988471Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1z",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Skipping Ambassador Svc",
"ts": 1567016109.9817085
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.981966362Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1y",
"jsonPayload": {
"msg": "Found identical Service",
"status": {
"loadBalancer": {}
},
"level": "info",
"logger": "seldon-controller",
"name": "seldon-5962968fa362e710c7798d69cd566fa6",
"namespace": "seldon",
"ts": 1567016109.9816587
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:15:09.981920464Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1x",
"jsonPayload": {
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor-istio"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor-istio"
},
{
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
]
},
{
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"protocol": "TCP",
"containerPort": 8082,
"name": "admin"
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
]
}
]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.967519700Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1w",
"jsonPayload": {
"containers": [{
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor-istio"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor-istio",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000
}
},
{
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"protocol": "TCP",
"containerPort": 8082,
"name": "admin"
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}]
},
"name": "podinfo"
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.967474193Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1v",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567016109.9672375,
"msg": "Updating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-8888366"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.967396143Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1u",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567016109.9667711
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.966848853Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1t",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567016109.9666095,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:09.966762263Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"insertId": "1qzzb5zg4niwa1s",
"jsonPayload": {
"ts": 1567016109.9664583,
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:15:09.966720861Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:15.652782047Z"
},
{
"textPayload": "2019/08/28 18:15:04 http: TLS handshake error from 10.24.0.29:40184: tls: first record does not look like a TLS handshake\n",
"insertId": "a9omqyg4nf6ewe",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:15:04.715660290Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:15:10.571800512Z"
},
{
"textPayload": "2019/08/28 18:14:49 http: TLS handshake error from 10.24.0.29:39996: tls: first record does not look like a TLS handshake\n",
"insertId": "i1od5vg4p07f1m",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:49.715415787Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:53.857929599Z"
},
{
"textPayload": "2019/08/28 18:14:34 http: TLS handshake error from 10.24.0.29:39804: tls: first record does not look like a TLS handshake\n",
"insertId": "1wjo8ejf5rwssj",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:34.715258772Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:40.749761266Z"
},
{
"textPayload": "2019/08/28 18:14:19 http: TLS handshake error from 10.24.0.29:39602: tls: first record does not look like a TLS handshake\n",
"insertId": "1bgmwg2fb07auz",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:14:19.715606793Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:25.654644777Z"
},
{
"insertId": "yz0h1hf5cl8su",
"jsonPayload": {
"msg": "Found identical Istio Destination Rule",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567016054.821958
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.822215302Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8st",
"jsonPayload": {
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc",
"namespace": "seldon",
"ts": 1567016054.8219388
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.822209628Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8ss",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567016054.8219075,
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.822204557Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sr",
"jsonPayload": {
"msg": "Skipping Ambassador Svc",
"ts": 1567016054.8218596,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.822199468Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sq",
"jsonPayload": {
"msg": "Found identical Service",
"status": {
"loadBalancer": {}
},
"level": "info",
"logger": "seldon-controller",
"name": "seldon-5962968fa362e710c7798d69cd566fa6",
"namespace": "seldon",
"ts": 1567016054.8218143
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.822163162Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sp",
"jsonPayload": {
"containers": [{
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File",
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"value": "sp-predictor-istio",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor-istio"
},
{
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000,
"procMount": "Default"
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
]
},
{
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler"
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:14:14.815084981Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8so",
"jsonPayload": {
"containers": [{
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor-istio",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"value": "sp-predictor-istio",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}]
},
{
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log"
},
{
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
},
"name": "podinfo"
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.814998937Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sn",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-8888366",
"namespace": "seldon",
"ts": 1567016054.8146899,
"msg": "Updating Deployment"
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:14:14.814838243Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sm",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567016054.8141816,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.814423476Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sl",
"jsonPayload": {
"ts": 1567016054.8139815,
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.814386109Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sk",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp",
"ts": 1567016054.8138008,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.814124181Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sj",
"jsonPayload": {
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567016054.7990687,
"msg": "Found identical Istio Destination Rule",
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:14:14.799245855Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8si",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567016054.7990417,
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.799241385Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sh",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567016054.79901,
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.799236660Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sg",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Skipping Ambassador Svc",
"ts": 1567016054.7989585
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.799231533Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sf",
"jsonPayload": {
"msg": "Found identical Service",
"level": "info",
"status": {
"loadBalancer": {}
},
"logger": "seldon-controller",
"name": "seldon-5962968fa362e710c7798d69cd566fa6",
"namespace": "seldon",
"ts": 1567016054.798898
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:14:14.799191454Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8se",
"jsonPayload": {
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}]
},
"name": "podinfo"
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor-istio",
"name": "PREDICTIVE_UNIT_ID"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"failureThreshold": 3,
"initialDelaySeconds": 20,
"timeoutSeconds": 1,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor-istio",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
]
},
{
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.778089529Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sd",
"jsonPayload": {
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"initialDelaySeconds": 60,
"timeoutSeconds": 1,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor-istio",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor-istio",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
}
},
{
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"protocol": "TCP",
"containerPort": 8082,
"name": "admin"
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
}
],
"imagePullSecrets": [{
"name": "ngc"
}]
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.777984600Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sc",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567016054.777805,
"msg": "Updating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-8888366"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.777878370Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sb",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567016054.7773554,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.777437142Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8sa",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567016054.777152,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.777226629Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s9",
"jsonPayload": {
"ts": 1567016054.776972,
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.777177040Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s8",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567016054.7147915,
"msg": "Found identical Istio Destination Rule"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.714916382Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s7",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567016054.7147663,
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.714910156Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s6",
"jsonPayload": {
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http",
"namespace": "seldon",
"ts": 1567016054.7147233
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.714902388Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s5",
"jsonPayload": {
"ts": 1567016054.7146244,
"level": "info",
"logger": "seldon-controller",
"msg": "Skipping Ambassador Svc"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.714848243Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s4",
"jsonPayload": {
"msg": "Found identical Service",
"level": "info",
"status": {
"loadBalancer": {}
},
"logger": "seldon-controller",
"name": "seldon-5962968fa362e710c7798d69cd566fa6",
"namespace": "seldon",
"ts": 1567016054.6890838
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.689677242Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s2",
"jsonPayload": {
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"name": "sp-predictor-istio",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File",
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor-istio",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"resources": {
"requests": {
"cpu": "2"
}
}
},
{
"securityContext": {
"procMount": "Default",
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log"
},
{
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"failureThreshold": 7,
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}]
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:14:14.681189266Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s3",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-8888366",
"namespace": "seldon",
"ts": 1567016054.6805422,
"msg": "Updating Deployment"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:14:14.681108322Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s1",
"jsonPayload": {
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor-istio",
"name": "PREDICTIVE_UNIT_ID"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor-istio",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
},
"limits": {
"nvidia.com/gpu": "1"
}
},
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
}
},
{
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
}
]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.681106372Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8s0",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567016054.679211
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:14:14.679708378Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rz",
"jsonPayload": {
"ts": 1567016054.6790762,
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:14:14.679702654Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8ry",
"jsonPayload": {
"ts": 1567016054.6789515,
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.679616165Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rx",
"jsonPayload": {
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567016054.6551552,
"msg": "Creating Istio Destination Rule",
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.655651116Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rw",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc",
"namespace": "seldon",
"ts": 1567016054.6282535,
"msg": "Creating Virtual Service"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.628506600Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rv",
"jsonPayload": {
"msg": "Creating Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http",
"namespace": "seldon",
"ts": 1567016054.5958114
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.596013302Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8ru",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Skipping Ambassador Svc",
"ts": 1567016054.595744
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.595965612Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rt",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567016054.5799978,
"msg": "Creating Service",
"level": "info",
"logger": "seldon-controller",
"name": "seldon-5962968fa362e710c7798d69cd566fa6"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.580210710Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rs",
"jsonPayload": {
"containers": [{
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor-istio",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"name": "sp-predictor-istio",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
}
},
{
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
}
},
{
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3Rvci1pc3RpbyIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yLWlzdGlvIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yLWlzdGlvIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}],
"defaultMode": 420
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.564798844Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rr",
"jsonPayload": {
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-8888366",
"namespace": "seldon",
"ts": 1567016054.5645704
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:14:14.564682560Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rq",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567016054.5641727,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.564271344Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rp",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567016054.5639367
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:14:14.564054465Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8ro",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp",
"ts": 1567016054.5635886,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:14.563786343Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"insertId": "yz0h1hf5cl8rn",
"jsonPayload": {
"graph": {
"implementation": "UNKNOWN_IMPLEMENTATION",
"endpoint": {
"type": "REST"
},
"parameters": [{
"name": "url",
"type": "STRING",
"value": "localhost:2001"
},
{
"name": "model_name",
"type": "STRING",
"value": "spelling_model"
},
{
"name": "protocol",
"type": "STRING",
"value": "grpc"
}
],
"name": "sp-predictor-istio",
"type": "MODEL"
},
"explainer": {
"containerSpec": {
"resources": {},
"name": ""
}
},
"componentSpecs": [{
"spec": {
"containers": [{
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor-istio"
},
{
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"command": [
"trtserver"
],
"name": "inference-server",
"securityContext": {
"runAsUser": 1000
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"terminationGracePeriodSeconds": 1
},
"metadata": {
"creationTimestamp": null
}
}],
"labels": {
"version": "v1",
"fluentd": "true"
},
"engineResources": {},
"replicas": 1,
"name": "nvidia-sp",
"svcOrchSpec": {
"resources": {
"requests": {
"cpu": "1"
}
}
}
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:14:14.556389015Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"textPayload": "predictor is now\n",
"insertId": "yz0h1hf5cl8rm",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:14:14.556297795Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:20.608135600Z"
},
{
"textPayload": "2019/08/28 18:14:04 http: TLS handshake error from 10.24.0.29:39392: tls: first record does not look like a TLS handshake\n",
"insertId": "ezhyv6g4oxuw4j",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:14:04.715518673Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:14:10.671393880Z"
},
{
"textPayload": "2019/08/28 18:13:49 http: TLS handshake error from 10.24.0.29:39200: tls: first record does not look like a TLS handshake\n",
"insertId": "1t7lrklg35cm9cn",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:13:49.715443668Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:13:56.460710245Z"
},
{
"textPayload": "2019/08/28 18:13:34 http: TLS handshake error from 10.24.0.29:39008: tls: first record does not look like a TLS handshake\n",
"insertId": "1ca2r59g5jo8cb3",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:13:34.715467301Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:13:41.305495023Z"
},
{
"textPayload": "2019/08/28 18:13:19 http: TLS handshake error from 10.24.0.29:38810: tls: first record does not look like a TLS handshake\n",
"insertId": "1ehznolg12ey8yr",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:13:19.715411678Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:13:26.202150978Z"
},
{
"textPayload": "2019/08/28 18:13:04 http: TLS handshake error from 10.24.0.29:38614: tls: first record does not look like a TLS handshake\n",
"insertId": "1rjtz63g3ltnms7",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:13:04.715460570Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:13:11.144947423Z"
},
{
"textPayload": "2019/08/28 18:12:49 http: TLS handshake error from 10.24.0.29:38426: tls: first record does not look like a TLS handshake\n",
"insertId": "lnhz64g4pnhrst",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:12:49.715702840Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:12:52.033355988Z"
},
{
"textPayload": "2019/08/28 18:12:34 http: TLS handshake error from 10.24.0.29:38236: tls: first record does not look like a TLS handshake\n",
"insertId": "tff9ehg4aqxcns",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:12:34.715320041Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:12:40.972751390Z"
},
{
"textPayload": "2019/08/28 18:12:19 http: TLS handshake error from 10.24.0.29:38038: tls: first record does not look like a TLS handshake\n",
"insertId": "1ohtt35g1wxh3sm",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:12:19.715496702Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:12:24.797871137Z"
},
{
"textPayload": "2019/08/28 18:12:04 http: TLS handshake error from 10.24.0.29:37844: tls: first record does not look like a TLS handshake\n",
"insertId": "qd9zsrg1tz69s0",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:12:04.715413868Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:12:10.587614841Z"
},
{
"textPayload": "2019/08/28 18:11:49 http: TLS handshake error from 10.24.0.29:37652: tls: first record does not look like a TLS handshake\n",
"insertId": "yz6l8zg4nq57fu",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:11:49.715268269Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:55.522489094Z"
},
{
"insertId": "9foptfg1dg0dqj",
"jsonPayload": {
"msg": "Found identical Istio Destination Rule",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567015901.2541704
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:11:41.254428857Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqi",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc",
"namespace": "seldon",
"ts": 1567015901.2541447,
"msg": "Found identical Virtual Service"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:11:41.254424555Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqh",
"jsonPayload": {
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http",
"namespace": "seldon",
"ts": 1567015901.2541108
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:11:41.254419528Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqg",
"jsonPayload": {
"ts": 1567015901.2540598,
"level": "info",
"logger": "seldon-controller",
"msg": "Skipping Ambassador Svc"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:11:41.254394800Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqf",
"jsonPayload": {
"msg": "Found identical Service",
"level": "info",
"status": {
"loadBalancer": {}
},
"logger": "seldon-controller",
"name": "seldon-8121bead1387ce5a8a62282502d48b99",
"namespace": "seldon",
"ts": 1567015901.2539964
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:11:41.254335118Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqe",
"jsonPayload": {
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
},
"name": "podinfo"
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"securityContext": {
"runAsUser": 1000,
"procMount": "Default"
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
},
"limits": {
"nvidia.com/gpu": "1"
}
},
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
},
{
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"protocol": "TCP",
"containerPort": 8082,
"name": "admin"
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:11:41.235128723Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqd",
"jsonPayload": {
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
},
{
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
}
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
}
],
"imagePullSecrets": [{
"name": "ngc"
}]
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:11:41.235048145Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqc",
"jsonPayload": {
"msg": "Updating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015901.2347517
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:11:41.234876688Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqb",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015901.2343526,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:11:41.234452172Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dqa",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567015901.2341712,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:11:41.234414803Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"insertId": "9foptfg1dg0dq9",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp",
"ts": 1567015901.234029,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:11:41.234371281Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:47.442403265Z"
},
{
"textPayload": "2019/08/28 18:11:34 http: TLS handshake error from 10.24.0.29:37462: tls: first record does not look like a TLS handshake\n",
"insertId": "m74wr7g5jv26gn",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:11:34.715784188Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:41.340844075Z"
},
{
"textPayload": "2019/08/28 18:11:19 http: TLS handshake error from 10.24.0.29:37264: tls: first record does not look like a TLS handshake\n",
"insertId": "j5qccig4nq2bf7",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:11:19.715459305Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:26.300222763Z"
},
{
"textPayload": "2019/08/28 18:11:04 http: TLS handshake error from 10.24.0.29:37068: tls: first record does not look like a TLS handshake\n",
"insertId": "rr5aqng5kesgyn",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:11:04.715261526Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:08.204934321Z"
},
{
"insertId": "1gqdzoxg4pboosw",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567015856.7378962,
"msg": "Found identical Istio Destination Rule"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:10:56.739529085Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosv",
"jsonPayload": {
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc",
"namespace": "seldon",
"ts": 1567015856.7378337
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.739491964Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosu",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015856.7377238,
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.737836306Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboost",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Skipping Ambassador Svc",
"ts": 1567015856.7376094
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.737803478Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pbooss",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015856.7374597,
"msg": "Found identical Service",
"level": "info",
"status": {
"loadBalancer": {}
},
"logger": "seldon-controller",
"name": "seldon-8121bead1387ce5a8a62282502d48b99"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.737750164Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosr",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015856.7082226,
"msg": "Updating Deployment"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.708729291Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosq",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015856.7077727,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.708725226Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosp",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567015856.7076337,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:10:56.708720847Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pbooso",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp",
"ts": 1567015856.7075138
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.708657768Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosn",
"jsonPayload": {
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"initialDelaySeconds": 60,
"timeoutSeconds": 1,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000,
"procMount": "Default"
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
]
},
{
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}]
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}]
}
}]
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:10:56.708643601Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosm",
"jsonPayload": {
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"livenessProbe": {
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"name": "sp-predictor"
},
{
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000
}
},
{
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"readinessProbe": {
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
}
}]
}
}]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.708543606Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosl",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567015856.6918097,
"msg": "Found identical Istio Destination Rule"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.692010963Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosk",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc",
"namespace": "seldon",
"ts": 1567015856.691782,
"msg": "Found identical Virtual Service"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:10:56.691975935Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosj",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http",
"namespace": "seldon",
"ts": 1567015856.6917484,
"msg": "Found identical Virtual Service"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.691971001Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosi",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Skipping Ambassador Svc",
"ts": 1567015856.6917021
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:10:56.691965351Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosh",
"jsonPayload": {
"status": {
"loadBalancer": {}
},
"level": "info",
"logger": "seldon-controller",
"name": "seldon-8121bead1387ce5a8a62282502d48b99",
"namespace": "seldon",
"ts": 1567015856.6916456,
"msg": "Found identical Service"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.691908219Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosg",
"jsonPayload": {
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"securityContext": {
"runAsUser": 1000,
"procMount": "Default"
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
},
"limits": {
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
},
{
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"items": [{
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
},
"path": "annotations"
}],
"defaultMode": 420
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:10:56.651908072Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosf",
"jsonPayload": {
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
},
"limits": {
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
]
},
{
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"readinessProbe": {
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}]
},
"name": "podinfo"
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.651657089Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboose",
"jsonPayload": {
"msg": "Updating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015856.6514325
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.651535221Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosd",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015856.6509662,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:10:56.651019901Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosc",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567015856.650749,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:10:56.650844935Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosb",
"jsonPayload": {
"ts": 1567015856.6504514,
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:10:56.650587706Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboosa",
"jsonPayload": {
"msg": "Found identical Istio Destination Rule",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567015856.6030214
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.604102655Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos9",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015856.6030037,
"msg": "Found identical Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:10:56.604096011Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos8",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http",
"namespace": "seldon",
"ts": 1567015856.6029687,
"msg": "Found identical Virtual Service"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.604088724Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos7",
"jsonPayload": {
"msg": "Skipping Ambassador Svc",
"ts": 1567015856.602896,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.604070796Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos6",
"jsonPayload": {
"level": "info",
"status": {
"loadBalancer": {}
},
"logger": "seldon-controller",
"name": "seldon-8121bead1387ce5a8a62282502d48b99",
"namespace": "seldon",
"ts": 1567015856.5888343,
"msg": "Found identical Service"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.588991408Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos5",
"jsonPayload": {
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"procMount": "Default",
"runAsUser": 1000
}
},
{
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
},
"path": "annotations"
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.578317901Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos4",
"jsonPayload": {
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"command": [
"trtserver"
],
"name": "inference-server"
},
{
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0=",
"name": "ENGINE_PREDICTOR"
},
{
"value": "nvidia-sp",
"name": "DEPLOYMENT_NAME"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
}
}
]
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:10:56.578217941Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos3",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015856.5780034,
"msg": "Updating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-27351ca"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:10:56.578096403Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos2",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567015856.5775406
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.577629135Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos1",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015856.5773125
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:10:56.577427288Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboos0",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp",
"ts": 1567015856.5770614,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:10:56.577217434Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboorz",
"jsonPayload": {
"msg": "Creating Istio Destination Rule",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-nvidia-sp",
"namespace": "seldon",
"ts": 1567015856.5612025
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.561375416Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboory",
"jsonPayload": {
"msg": "Creating Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-grpc",
"namespace": "seldon",
"ts": 1567015856.5309663
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.531195199Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboorx",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015856.5026171,
"msg": "Creating Virtual Service",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-http"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.502782459Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboorw",
"jsonPayload": {
"msg": "Skipping Ambassador Svc",
"ts": 1567015856.502546,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.502747643Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboorv",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "seldon-8121bead1387ce5a8a62282502d48b99",
"namespace": "seldon",
"ts": 1567015856.476257,
"msg": "Creating Service"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:10:56.476654118Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pbooru",
"jsonPayload": {
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"livenessProbe": {
"failureThreshold": 3,
"initialDelaySeconds": 60,
"timeoutSeconds": 1,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"initialDelaySeconds": 20,
"timeoutSeconds": 1,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor"
},
{
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
}
},
{
"livenessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7ImxpbWl0cyI6eyJudmlkaWEuY29tL2dwdSI6IjEifSwicmVxdWVzdHMiOnsiY3B1IjoiMSIsIm52aWRpYS5jb20vZ3B1IjoiMSJ9fSwidGVybWluYXRpb25NZXNzYWdlUGF0aCI6Ii9kZXYvdGVybWluYXRpb24tbG9nIiwidGVybWluYXRpb25NZXNzYWdlUG9saWN5IjoiRmlsZSIsImltYWdlUHVsbFBvbGljeSI6IklmTm90UHJlc2VudCIsInNlY3VyaXR5Q29udGV4dCI6eyJydW5Bc1VzZXIiOjEwMDB9fV0sInRlcm1pbmF0aW9uR3JhY2VQZXJpb2RTZWNvbmRzIjoxLCJpbWFnZVB1bGxTZWNyZXRzIjpbeyJuYW1lIjoibmdjIn1dfX1dLCJyZXBsaWNhcyI6MSwiZW5naW5lUmVzb3VyY2VzIjp7fSwibGFiZWxzIjp7ImZsdWVudGQiOiJ0cnVlIiwidmVyc2lvbiI6InYxIn0sInN2Y09yY2hTcGVjIjp7InJlc291cmNlcyI6eyJyZXF1ZXN0cyI6eyJjcHUiOiIxIn19fSwiZXhwbGFpbmVyIjp7ImNvbnRhaW5lclNwZWMiOnsibmFtZSI6IiIsInJlc291cmNlcyI6e319fX0="
},
{
"name": "DEPLOYMENT_NAME",
"value": "nvidia-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
}
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
}
}
]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.461724282Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboort",
"jsonPayload": {
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "nvidia-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015856.4612646
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:10:56.461454886Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboors",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567015856.4608235
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.460937837Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboorr",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015856.4606235
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.460741420Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboorq",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-nvidia-sp-nvidia-sp",
"ts": 1567015856.460428,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.460710757Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"insertId": "1gqdzoxg4pboorp",
"jsonPayload": {
"labels": {
"version": "v1",
"fluentd": "true"
},
"engineResources": {},
"replicas": 1,
"name": "nvidia-sp",
"svcOrchSpec": {
"resources": {
"requests": {
"cpu": "1"
}
}
},
"graph": {
"type": "MODEL",
"implementation": "UNKNOWN_IMPLEMENTATION",
"endpoint": {
"type": "REST"
},
"parameters": [{
"name": "url",
"type": "STRING",
"value": "localhost:2001"
},
{
"name": "model_name",
"type": "STRING",
"value": "spelling_model"
},
{
"name": "protocol",
"type": "STRING",
"value": "grpc"
}
],
"name": "sp-predictor"
},
"explainer": {
"containerSpec": {
"resources": {},
"name": ""
}
},
"componentSpecs": [{
"spec": {
"containers": [{
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor"
},
{
"name": "inference-server",
"command": [
"trtserver"
],
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"limits": {
"nvidia.com/gpu": "1"
},
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"terminationGracePeriodSeconds": 1
},
"metadata": {
"creationTimestamp": null
}
}]
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.451494339Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"textPayload": "predictor is now\n",
"insertId": "1gqdzoxg4pbooro",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:56.451153429Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:11:02.099703981Z"
},
{
"textPayload": "2019/08/28 18:10:49 http: TLS handshake error from 10.24.0.29:36872: tls: first record does not look like a TLS handshake\n",
"insertId": "1wjo8ejf5r2hhe",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:10:49.715725598Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:10:56.272551559Z"
},
{
"textPayload": "2019/08/28 18:10:34 http: TLS handshake error from 10.24.0.29:36680: tls: first record does not look like a TLS handshake\n",
"insertId": "1inwm42g4l9bsem",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:10:34.715721483Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:10:41.183748434Z"
},
{
"textPayload": "2019/08/28 18:10:19 http: TLS handshake error from 10.24.0.29:36492: tls: first record does not look like a TLS handshake\n",
"insertId": "1inw11of5uyo6w",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:10:19.715810386Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:10:26.098578183Z"
},
{
"textPayload": "2019/08/28 18:10:04 http: TLS handshake error from 10.24.0.29:36292: tls: first record does not look like a TLS handshake\n",
"insertId": "1inutu6f61x0p7",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:10:04.715599631Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:10:10.915969195Z"
},
{
"textPayload": "2019/08/28 18:09:49 http: TLS handshake error from 10.24.0.29:36102: tls: first record does not look like a TLS handshake\n",
"insertId": "k5tp3g3s21swg",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:09:49.715576297Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:09:54.811649760Z"
},
{
"textPayload": "2019/08/28 18:09:34 http: TLS handshake error from 10.24.0.29:35910: tls: first record does not look like a TLS handshake\n",
"insertId": "nl4qofg478twpa",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:09:34.715534024Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:09:41.129678757Z"
},
{
"textPayload": "2019/08/28 18:09:19 http: TLS handshake error from 10.24.0.29:35724: tls: first record does not look like a TLS handshake\n",
"insertId": "jfbfagg5jxfr5s",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:09:19.715327628Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:09:25.224788817Z"
},
{
"textPayload": "2019/08/28 18:09:04 http: TLS handshake error from 10.24.0.29:35520: tls: first record does not look like a TLS handshake\n",
"insertId": "1033ic0g4n9dcpl",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:09:04.715551473Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:09:11.148455603Z"
},
{
"textPayload": "2019/08/28 18:08:49 http: TLS handshake error from 10.24.0.29:35336: tls: first record does not look like a TLS handshake\n",
"insertId": "1k1sloag4oavffr",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:08:49.715993744Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:56.153641316Z"
},
{
"textPayload": "2019/08/28 18:08:34 http: TLS handshake error from 10.24.0.29:35144: tls: first record does not look like a TLS handshake\n",
"insertId": "ezgleaf8f2s74",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:08:34.715373506Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:39.048871680Z"
},
{
"insertId": "ezgleaf8f2s73",
"jsonPayload": {
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015713.0055113,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources"
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:08:33.005741439Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:39.048871680Z"
},
{
"insertId": "ezgleaf8f2s72",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015713.001493,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:08:33.003350151Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:39.048871680Z"
},
{
"insertId": "ezgleaf8f2s71",
"jsonPayload": {
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"items": [{
"path": "annotations",
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
}
}],
"defaultMode": 420
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File",
"livenessProbe": {
"initialDelaySeconds": 60,
"timeoutSeconds": 1,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}]
},
{
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
}
},
{
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19",
"name": "ENGINE_PREDICTOR"
},
{
"value": "trt-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine"
}
],
"imagePullSecrets": [{
"name": "ngc"
}]
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:08:33.003201920Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:39.048871680Z"
},
{
"insertId": "ezgleaf8f2s70",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567015713.0010517
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:08:33.001354435Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:39.048871680Z"
},
{
"insertId": "ezgleaf8f2s6z",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015713.0007417
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:08:33.001344084Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:39.048871680Z"
},
{
"insertId": "ezgleaf8f2s6y",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015712.999806,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:08:33.001190288Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:39.048871680Z"
},
{
"textPayload": "2019/08/28 18:08:19 http: TLS handshake error from 10.24.0.29:34958: tls: first record does not look like a TLS handshake\n",
"insertId": "q3b92hg4vbtek5",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:08:19.715373758Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:24.906176693Z"
},
{
"textPayload": "2019/08/28 18:08:04 http: TLS handshake error from 10.24.0.29:34750: tls: first record does not look like a TLS handshake\n",
"insertId": "n1ffpzg14cra68",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:08:04.715255191Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:08:10.844587005Z"
},
{
"insertId": "59yttjg40s17y7",
"jsonPayload": {
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015672.0394876,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:52.039765445Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:55.751572855Z"
},
{
"insertId": "59yttjg40s17y6",
"jsonPayload": {
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File",
"livenessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"initialDelaySeconds": 60,
"timeoutSeconds": 1,
"successThreshold": 1
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"resources": {
"requests": {
"cpu": "2"
}
}
},
{
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
},
{
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}],
"defaultMode": 420
}
}]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:51.844618154Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:55.751572855Z"
},
{
"insertId": "59yttjg40s17y5",
"jsonPayload": {
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015671.8442202,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:51.844323704Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:55.751572855Z"
},
{
"insertId": "59yttjg40s17y4",
"jsonPayload": {
"ts": 1567015671.843928,
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config"
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:07:51.844119374Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:55.751572855Z"
},
{
"insertId": "59yttjg40s17y3",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015671.8437421
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:51.843924505Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:55.751572855Z"
},
{
"insertId": "59yttjg40s17y2",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015671.8435752
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:07:51.843874165Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:55.751572855Z"
},
{
"textPayload": "2019/08/28 18:07:49 http: TLS handshake error from 10.24.0.29:34560: tls: first record does not look like a TLS handshake\n",
"insertId": "59yttjg40s17y1",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:49.715610367Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:55.751572855Z"
},
{
"textPayload": "2019/08/28 18:07:34 http: TLS handshake error from 10.24.0.29:34372: tls: first record does not look like a TLS handshake\n",
"insertId": "ozld5cg1h4gem1",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:34.715454177Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:36.651346667Z"
},
{
"insertId": "ozld5cg1h4gem0",
"jsonPayload": {
"ts": 1567015651.3632684,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:31.363556933Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:36.651346667Z"
},
{
"insertId": "ozld5cg1h4gely",
"jsonPayload": {
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015651.3545077,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:31.356254283Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:36.651346667Z"
},
{
"insertId": "ozld5cg1h4gelz",
"jsonPayload": {
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File",
"livenessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"resources": {
"requests": {
"cpu": "2"
}
}
},
{
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log"
},
{
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"value": "trt-sp",
"name": "DEPLOYMENT_NAME"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
}
}]
}
}]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:31.354886570Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:36.651346667Z"
},
{
"insertId": "ozld5cg1h4gelx",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015651.3541467,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:31.354539160Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:36.651346667Z"
},
{
"insertId": "ozld5cg1h4gelw",
"jsonPayload": {
"ts": 1567015651.3539295,
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:31.354168806Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:36.651346667Z"
},
{
"insertId": "ozld5cg1h4gelv",
"jsonPayload": {
"ts": 1567015651.3537736,
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:31.354097389Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:36.651346667Z"
},
{
"insertId": "1nxn5t7f5lxb7j",
"jsonPayload": {
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"request": "seldon/nvidia-sp",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015641.1134713
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:21.113757832Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:25.589539842Z"
},
{
"insertId": "1nxn5t7f5lxb7i",
"jsonPayload": {
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
}
},
{
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"value": "trt-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}]
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:21.109264786Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:25.589539842Z"
},
{
"insertId": "1nxn5t7f5lxb7h",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015641.1089573,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:21.109064747Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:25.589539842Z"
},
{
"insertId": "1nxn5t7f5lxb7g",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015641.1086357,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:21.108756119Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:25.589539842Z"
},
{
"insertId": "1nxn5t7f5lxb7f",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015641.108436
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:21.108531352Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:25.589539842Z"
},
{
"insertId": "1nxn5t7f5lxb7e",
"jsonPayload": {
"ts": 1567015641.1081924,
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:21.108418062Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:25.589539842Z"
},
{
"textPayload": "2019/08/28 18:07:19 http: TLS handshake error from 10.24.0.29:34186: tls: first record does not look like a TLS handshake\n",
"insertId": "1nxn5t7f5lxb7d",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:19.715430586Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:25.589539842Z"
},
{
"insertId": "1gg939fg415pntx",
"jsonPayload": {
"ts": 1567015635.9879234,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"request": "seldon/nvidia-sp",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:15.988352225Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:20.573795964Z"
},
{
"insertId": "1gg939fg415pntw",
"jsonPayload": {
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3,
"initialDelaySeconds": 60,
"timeoutSeconds": 1,
"successThreshold": 1
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"initialDelaySeconds": 20,
"timeoutSeconds": 1,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
},
{
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"failureThreshold": 7,
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19",
"name": "ENGINE_PREDICTOR"
},
{
"value": "trt-sp",
"name": "DEPLOYMENT_NAME"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine"
}
],
"imagePullSecrets": [{
"name": "ngc"
}]
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:07:15.984339859Z",
"severity": "INFO",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:20.573795964Z"
},
{
"insertId": "1gg939fg415pntv",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015635.9840631,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:15.984221432Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:20.573795964Z"
},
{
"insertId": "1gg939fg415pntu",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015635.9837115,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:15.984020925Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:20.573795964Z"
},
{
"insertId": "1gg939fg415pntt",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015635.9835181
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:15.983791549Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:20.573795964Z"
},
{
"insertId": "1gg939fg415pnts",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015635.9832542,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:15.983745515Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:20.573795964Z"
},
{
"insertId": "17aogfvfel78zz",
"jsonPayload": {
"ts": 1567015633.4229667,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"request": "seldon/nvidia-sp",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:13.423339420Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zy",
"jsonPayload": {
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"nvidia.com/gpu": "1",
"cpu": "1"
}
},
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
}
},
{
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:13.419057566Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zx",
"jsonPayload": {
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015633.4188218
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:13.418954343Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zw",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015633.4185276,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:13.418821021Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zv",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567015633.4183292,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:13.418604051Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zu",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015633.4181406,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:07:13.418533385Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zt",
"jsonPayload": {
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015632.137884,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"request": "seldon/nvidia-sp",
"msg": "Reconciler error"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:12.138212840Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zs",
"jsonPayload": {
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015632.134123
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:07:12.134450840Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zr",
"jsonPayload": {
"restartPolicy": "Always",
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}]
},
"name": "podinfo"
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
]
},
{
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:12.134374684Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zq",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015632.133832,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:12.134038278Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zp",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567015632.1336477,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:12.133798902Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zo",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015632.1334722,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:12.133741923Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zn",
"jsonPayload": {
"ts": 1567015631.1330802,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"request": "seldon/nvidia-sp",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:11.133434433Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zm",
"jsonPayload": {
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
},
"name": "podinfo"
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
},
{
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server"
},
{
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"initialDelaySeconds": 20,
"timeoutSeconds": 2,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
}
}
]
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:11.125231454Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zl",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015631.1244347,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:11.124528158Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zk",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015631.124098,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:11.124175513Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zj",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015631.1239002
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:11.124158441Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zi",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015631.123724,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:11.124021760Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zh",
"jsonPayload": {
"ts": 1567015630.1234014,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:10.123622782Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zg",
"jsonPayload": {
"containers": [{
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"protocol": "TCP",
"containerPort": 9000,
"name": "http"
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
},
"failureThreshold": 3
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log"
},
{
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:10.120106137Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zf",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015630.1197114,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:10.119908487Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78ze",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015630.1194396,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:10.119538334Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zd",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015630.1192136
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:10.119485364Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zc",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015630.118997
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:10.119425167Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78zb",
"jsonPayload": {
"ts": 1567015629.1185582,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"request": "seldon/nvidia-sp",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:09.118823705Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78za",
"jsonPayload": {
"containers": [{
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File",
"livenessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}]
},
{
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log"
},
{
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"value": "trt-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"readinessProbe": {
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
}
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
},
"name": "podinfo"
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:09.113957334Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78z9",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015629.1137288,
"msg": "Creating Deployment"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:09.113862910Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78z8",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567015629.1134627
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:09.113578081Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78z7",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015629.1132276
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:09.113407597Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "17aogfvfel78z6",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015629.1129835
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:09.113206623Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:14.713306870Z"
},
{
"insertId": "o5k4lnfh9xqka",
"jsonPayload": {
"ts": 1567015628.1126077,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:08.112868991Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk9",
"jsonPayload": {
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
},
{
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"protocol": "TCP",
"containerPort": 8082,
"name": "admin"
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"procMount": "Default",
"runAsUser": 8888
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"value": "trt-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"readinessProbe": {
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"securityContext": {},
"dnsPolicy": "ClusterFirst",
"terminationGracePeriodSeconds": 20
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:08.109118040Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk8",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015628.1088748,
"msg": "Creating Deployment"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:08.109090392Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk7",
"jsonPayload": {
"ts": 1567015628.1086156,
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:08.108763592Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk6",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015628.1084368
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:08.108594180Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk5",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015628.1081848,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:07:08.108532995Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk4",
"jsonPayload": {
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015627.1077707,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"msg": "Reconciler error"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:07.108060917Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk3",
"jsonPayload": {
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
}
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server",
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000
}
},
{
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19",
"name": "ENGINE_PREDICTOR"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"readinessProbe": {
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
}
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"protocol": "TCP",
"containerPort": 8082,
"name": "admin"
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
}
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always"
},
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0"
}
},
"timestamp": "2019-08-28T18:07:07.102047755Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk2",
"jsonPayload": {
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015627.1017475
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:07.101858875Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk1",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567015627.1014714
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:07.101588272Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqk0",
"jsonPayload": {
"ts": 1567015627.1012115,
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:07.101367718Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjz",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015627.100964,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:07.101186958Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjy",
"jsonPayload": {
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015626.1004703,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"request": "seldon/nvidia-sp",
"msg": "Reconciler error"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:06.100686328Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjx",
"jsonPayload": {
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"fieldPath": "metadata.annotations",
"apiVersion": "v1"
},
"path": "annotations"
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler",
"containers": [{
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"value": "nvidia-sp",
"name": "SELDON_DEPLOYMENT_ID"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "sp-predictor",
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePolicy": "File"
},
{
"name": "inference-server",
"command": [
"trtserver"
],
"terminationMessagePolicy": "File",
"terminationMessagePath": "/dev/termination-log",
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent"
},
{
"resources": {
"requests": {
"cpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"name": "ENGINE_SERVER_GRPC_PORT",
"value": "5001"
},
{
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1",
"name": "JAVA_OPTS"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
]
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:06.096750557Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjw",
"jsonPayload": {
"namespace": "seldon",
"ts": 1567015626.096524,
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:06.096619818Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjv",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Creating default Ambassador config",
"ts": 1567015626.0962136
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:06.096336898Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqju",
"jsonPayload": {
"ts": 1567015626.096009,
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:06.096254163Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjt",
"jsonPayload": {
"ts": 1567015626.0958014,
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:06.096053242Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjs",
"jsonPayload": {
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"msg": "Reconciler error",
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015625.0953524,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:05.095632141Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjr",
"jsonPayload": {
"containers": [{
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"livenessProbe": {
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5,
"failureThreshold": 3
},
"env": [{
"value": "9000",
"name": "PREDICTIVE_UNIT_SERVICE_PORT"
},
{
"value": "sp-predictor",
"name": "PREDICTIVE_UNIT_ID"
},
{
"name": "PREDICTOR_ID",
"value": "nvidia-sp"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"name": "PREDICTIVE_UNIT_PARAMETERS",
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}]
},
{
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000
},
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"name": "inference-server",
"command": [
"trtserver"
]
},
{
"name": "seldon-container-engine",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5,
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
}
},
"env": [{
"name": "ENGINE_PREDICTOR",
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19"
},
{
"name": "DEPLOYMENT_NAME",
"value": "trt-sp"
},
{
"value": "seldon",
"name": "DEPLOYMENT_NAMESPACE"
},
{
"value": "8000",
"name": "ENGINE_SERVER_PORT"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"value": "false",
"name": "SELDON_LOG_MESSAGES_EXTERNALLY"
}
],
"resources": {
"requests": {
"cpu": "1"
}
},
"readinessProbe": {
"failureThreshold": 1,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
}
},
"imagePullPolicy": "IfNotPresent",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"volumeMounts": [{
"mountPath": "/etc/podinfo",
"name": "podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"protocol": "TCP",
"containerPort": 9090,
"name": "jmx"
}
],
"image": "docker.io/seldonio/engine:0.4.0"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"downwardAPI": {
"defaultMode": 420,
"items": [{
"path": "annotations",
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
}
}]
},
"name": "podinfo"
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20,
"schedulerName": "default-scheduler"
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:05.091630889Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjq",
"jsonPayload": {
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015625.091338
},
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:07:05.091464078Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjp",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015625.0909433,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:05.091035617Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjo",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "Not creating container service for inference-server",
"ts": 1567015625.0907648
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:05.090850192Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjn",
"jsonPayload": {
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015625.0905623,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:05.090729757Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"textPayload": "2019/08/28 18:07:04 http: TLS handshake error from 10.24.0.29:33980: tls: first record does not look like a TLS handshake\n",
"insertId": "o5k4lnfh9xqjm",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:04.715249061Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjl",
"jsonPayload": {
"level": "error",
"logger": "kubebuilder.controller",
"ts": 1567015624.090159,
"controller": "seldondeployment-controller",
"stacktrace": "github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr.(*zapLogger).Error\n\t/go/src/github.com/seldonio/seldon-operator/vendor/github.com/go-logr/zapr/zapr.go:128\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:217\ngithub.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/sigs.k8s.io/controller-runtime/pkg/internal/controller/controller.go:158\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:134\ngithub.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait.Until\n\t/go/src/github.com/seldonio/seldon-operator/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:88",
"request": "seldon/nvidia-sp",
"error": "Deployment.apps \"trt-sp-nvidia-sp-27351ca\" is invalid: spec.template.spec.containers[1].resources.limits: Required value: Limit must be set for non overcommitable resources",
"msg": "Reconciler error"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:04.090487272Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjk",
"jsonPayload": {
"schedulerName": "default-scheduler",
"containers": [{
"env": [{
"name": "PREDICTIVE_UNIT_SERVICE_PORT",
"value": "9000"
},
{
"name": "PREDICTIVE_UNIT_ID",
"value": "sp-predictor"
},
{
"value": "nvidia-sp",
"name": "PREDICTOR_ID"
},
{
"name": "SELDON_DEPLOYMENT_ID",
"value": "nvidia-sp"
},
{
"value": "[{\"name\":\"url\",\"value\":\"localhost:2001\",\"type\":\"STRING\"},{\"name\":\"model_name\",\"value\":\"spelling_model\",\"type\":\"STRING\"},{\"name\":\"protocol\",\"value\":\"grpc\",\"type\":\"STRING\"}]",
"name": "PREDICTIVE_UNIT_PARAMETERS"
}
],
"ports": [{
"name": "http",
"protocol": "TCP",
"containerPort": 9000
}],
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"imagePullPolicy": "IfNotPresent",
"readinessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 20,
"successThreshold": 1,
"tcpSocket": {
"port": "http"
},
"periodSeconds": 5
},
"name": "sp-predictor",
"terminationMessagePolicy": "File",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"/bin/sleep 10"
]
}
}
},
"terminationMessagePath": "/dev/termination-log",
"livenessProbe": {
"failureThreshold": 3,
"timeoutSeconds": 1,
"initialDelaySeconds": 60,
"successThreshold": 1,
"periodSeconds": 5,
"tcpSocket": {
"port": "http"
}
}
},
{
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"imagePullPolicy": "IfNotPresent",
"command": [
"trtserver"
],
"name": "inference-server"
},
{
"securityContext": {
"runAsUser": 8888,
"procMount": "Default"
},
"livenessProbe": {
"httpGet": {
"path": "/live",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 7,
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 5
},
"env": [{
"value": "eyJuYW1lIjoibnZpZGlhLXNwIiwiZ3JhcGgiOnsibmFtZSI6InNwLXByZWRpY3RvciIsInR5cGUiOiJNT0RFTCIsImltcGxlbWVudGF0aW9uIjoiVU5LTk9XTl9JTVBMRU1FTlRBVElPTiIsImVuZHBvaW50Ijp7InNlcnZpY2VfaG9zdCI6ImxvY2FsaG9zdCIsInNlcnZpY2VfcG9ydCI6OTAwMCwidHlwZSI6IlJFU1QifSwicGFyYW1ldGVycyI6W3sibmFtZSI6InVybCIsInZhbHVlIjoibG9jYWxob3N0OjIwMDEiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoibW9kZWxfbmFtZSIsInZhbHVlIjoic3BlbGxpbmdfbW9kZWwiLCJ0eXBlIjoiU1RSSU5HIn0seyJuYW1lIjoicHJvdG9jb2wiLCJ2YWx1ZSI6ImdycGMiLCJ0eXBlIjoiU1RSSU5HIn1dfSwiY29tcG9uZW50U3BlY3MiOlt7Im1ldGFkYXRhIjp7ImNyZWF0aW9uVGltZXN0YW1wIjpudWxsfSwic3BlYyI6eyJjb250YWluZXJzIjpbeyJuYW1lIjoic3AtcHJlZGljdG9yIiwiaW1hZ2UiOiJnY3IuaW8vZ24tZGF0YS1zY2llbmNlLXByb2plY3QwMi9kZXYtbW9kZWxzL3NwLXByZWRpY3RvcjowLjMiLCJwb3J0cyI6W3sibmFtZSI6Imh0dHAiLCJjb250YWluZXJQb3J0Ijo5MDAwLCJwcm90b2NvbCI6IlRDUCJ9XSwiZW52IjpbeyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1NFUlZJQ0VfUE9SVCIsInZhbHVlIjoiOTAwMCJ9LHsibmFtZSI6IlBSRURJQ1RJVkVfVU5JVF9JRCIsInZhbHVlIjoic3AtcHJlZGljdG9yIn0seyJuYW1lIjoiUFJFRElDVE9SX0lEIiwidmFsdWUiOiJudmlkaWEtc3AifSx7Im5hbWUiOiJTRUxET05fREVQTE9ZTUVOVF9JRCIsInZhbHVlIjoibnZpZGlhLXNwIn0seyJuYW1lIjoiUFJFRElDVElWRV9VTklUX1BBUkFNRVRFUlMiLCJ2YWx1ZSI6Ilt7XCJuYW1lXCI6XCJ1cmxcIixcInZhbHVlXCI6XCJsb2NhbGhvc3Q6MjAwMVwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcIm1vZGVsX25hbWVcIixcInZhbHVlXCI6XCJzcGVsbGluZ19tb2RlbFwiLFwidHlwZVwiOlwiU1RSSU5HXCJ9LHtcIm5hbWVcIjpcInByb3RvY29sXCIsXCJ2YWx1ZVwiOlwiZ3JwY1wiLFwidHlwZVwiOlwiU1RSSU5HXCJ9XSJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjIifX0sImxpdmVuZXNzUHJvYmUiOnsidGNwU29ja2V0Ijp7InBvcnQiOiJodHRwIn0sImluaXRpYWxEZWxheVNlY29uZHMiOjYwLCJ0aW1lb3V0U2Vjb25kcyI6MSwicGVyaW9kU2Vjb25kcyI6NSwic3VjY2Vzc1RocmVzaG9sZCI6MSwiZmFpbHVyZVRocmVzaG9sZCI6M30sInJlYWRpbmVzc1Byb2JlIjp7InRjcFNvY2tldCI6eyJwb3J0IjoiaHR0cCJ9LCJpbml0aWFsRGVsYXlTZWNvbmRzIjoyMCwidGltZW91dFNlY29uZHMiOjEsInBlcmlvZFNlY29uZHMiOjUsInN1Y2Nlc3NUaHJlc2hvbGQiOjEsImZhaWx1cmVUaHJlc2hvbGQiOjN9LCJsaWZlY3ljbGUiOnsicHJlU3RvcCI6eyJleGVjIjp7ImNvbW1hbmQiOlsiL2Jpbi9zaCIsIi1jIiwiL2Jpbi9zbGVlcCAxMCJdfX19LCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQYXRoIjoiL2Rldi90ZXJtaW5hdGlvbi1sb2ciLCJ0ZXJtaW5hdGlvbk1lc3NhZ2VQb2xpY3kiOiJGaWxlIiwiaW1hZ2VQdWxsUG9saWN5IjoiSWZOb3RQcmVzZW50In0seyJuYW1lIjoiaW5mZXJlbmNlLXNlcnZlciIsImltYWdlIjoibnZjci5pby9udmlkaWEvdGVuc29ycnRzZXJ2ZXI6MTkuMDctcHkzIiwiY29tbWFuZCI6WyJ0cnRzZXJ2ZXIiXSwiYXJncyI6WyItLW1vZGVsLXN0b3JlPWdzOi8vZGV2LW1vZGVscy1kc2cvc2VhcmNoL3NwZWxsaW5nLW1vZGVscy0yNTYiLCItLWh0dHAtcG9ydD0yMDAwIiwiLS1ncnBjLXBvcnQ9MjAwMSJdLCJwb3J0cyI6W3siY29udGFpbmVyUG9ydCI6MjAwMCwicHJvdG9jb2wiOiJUQ1AifSx7ImNvbnRhaW5lclBvcnQiOjIwMDEsInByb3RvY29sIjoiVENQIn0seyJjb250YWluZXJQb3J0IjoyMDAyLCJwcm90b2NvbCI6IlRDUCJ9XSwicmVzb3VyY2VzIjp7InJlcXVlc3RzIjp7ImNwdSI6IjEiLCJudmlkaWEuY29tL2dwdSI6IjEifX0sInRlcm1pbmF0aW9uTWVzc2FnZVBhdGgiOiIvZGV2L3Rlcm1pbmF0aW9uLWxvZyIsInRlcm1pbmF0aW9uTWVzc2FnZVBvbGljeSI6IkZpbGUiLCJpbWFnZVB1bGxQb2xpY3kiOiJJZk5vdFByZXNlbnQiLCJzZWN1cml0eUNvbnRleHQiOnsicnVuQXNVc2VyIjoxMDAwfX1dLCJ0ZXJtaW5hdGlvbkdyYWNlUGVyaW9kU2Vjb25kcyI6MSwiaW1hZ2VQdWxsU2VjcmV0cyI6W3sibmFtZSI6Im5nYyJ9XX19XSwicmVwbGljYXMiOjEsImVuZ2luZVJlc291cmNlcyI6e30sImxhYmVscyI6eyJmbHVlbnRkIjoidHJ1ZSIsInZlcnNpb24iOiJ2MSJ9LCJzdmNPcmNoU3BlYyI6eyJyZXNvdXJjZXMiOnsicmVxdWVzdHMiOnsiY3B1IjoiMSJ9fX0sImV4cGxhaW5lciI6eyJjb250YWluZXJTcGVjIjp7Im5hbWUiOiIiLCJyZXNvdXJjZXMiOnt9fX19",
"name": "ENGINE_PREDICTOR"
},
{
"value": "trt-sp",
"name": "DEPLOYMENT_NAME"
},
{
"name": "DEPLOYMENT_NAMESPACE",
"value": "seldon"
},
{
"name": "ENGINE_SERVER_PORT",
"value": "8000"
},
{
"value": "5001",
"name": "ENGINE_SERVER_GRPC_PORT"
},
{
"name": "JAVA_OPTS",
"value": "-server -Dcom.sun.management.jmxremote.rmi.port=9090 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9090 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1"
},
{
"name": "SELDON_LOG_MESSAGES_EXTERNALLY",
"value": "false"
}
],
"readinessProbe": {
"timeoutSeconds": 2,
"initialDelaySeconds": 20,
"successThreshold": 1,
"periodSeconds": 1,
"httpGet": {
"path": "/ready",
"port": "admin",
"scheme": "HTTP"
},
"failureThreshold": 1
},
"imagePullPolicy": "IfNotPresent",
"resources": {
"requests": {
"cpu": "1"
}
},
"terminationMessagePath": "/dev/termination-log",
"lifecycle": {
"preStop": {
"exec": {
"command": [
"/bin/sh",
"-c",
"curl 127.0.0.1:8000/pause; /bin/sleep 10"
]
}
}
},
"volumeMounts": [{
"name": "podinfo",
"mountPath": "/etc/podinfo"
}],
"ports": [{
"protocol": "TCP",
"containerPort": 8000
},
{
"protocol": "TCP",
"containerPort": 5001
},
{
"name": "admin",
"protocol": "TCP",
"containerPort": 8082
},
{
"name": "jmx",
"protocol": "TCP",
"containerPort": 9090
}
],
"image": "docker.io/seldonio/engine:0.4.0",
"name": "seldon-container-engine",
"terminationMessagePolicy": "File"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"restartPolicy": "Always",
"volumes": [{
"name": "podinfo",
"downwardAPI": {
"defaultMode": 420,
"items": [{
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "metadata.annotations"
},
"path": "annotations"
}]
}
}],
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"terminationGracePeriodSeconds": 20
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:04.077977903Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjj",
"jsonPayload": {
"msg": "Creating Deployment",
"level": "info",
"logger": "seldon-controller",
"name": "trt-sp-nvidia-sp-27351ca",
"namespace": "seldon",
"ts": 1567015624.0775015
},
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:04.077587169Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqji",
"jsonPayload": {
"msg": "Creating default Ambassador config",
"ts": 1567015624.0771904,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:04.077389792Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjh",
"jsonPayload": {
"msg": "Not creating container service for inference-server",
"ts": 1567015624.0770233,
"level": "info",
"logger": "seldon-controller"
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:04.077326227Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjg",
"jsonPayload": {
"level": "info",
"logger": "seldon-controller",
"msg": "pSvcName",
"val": "nvidia-sp-trt-sp-nvidia-sp",
"ts": 1567015624.0768204
},
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:04.077129241Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"insertId": "o5k4lnfh9xqjf",
"jsonPayload": {
"explainer": {
"containerSpec": {
"resources": {},
"name": ""
}
},
"componentSpecs": [{
"metadata": {
"creationTimestamp": null
},
"spec": {
"containers": [{
"image": "gcr.io/gn-data-science-project02/dev-models/sp-predictor:0.3",
"resources": {
"requests": {
"cpu": "2"
}
},
"name": "sp-predictor"
},
{
"securityContext": {
"runAsUser": 1000
},
"ports": [{
"protocol": "TCP",
"containerPort": 2000
},
{
"protocol": "TCP",
"containerPort": 2001
},
{
"protocol": "TCP",
"containerPort": 2002
}
],
"args": [
"--model-store=gs://dev-models-dsg/search/spelling-models-256",
"--http-port=2000",
"--grpc-port=2001"
],
"image": "nvcr.io/nvidia/tensorrtserver:19.07-py3",
"resources": {
"requests": {
"cpu": "1",
"nvidia.com/gpu": "1"
}
},
"command": [
"trtserver"
],
"name": "inference-server"
}
],
"imagePullSecrets": [{
"name": "ngc"
}],
"terminationGracePeriodSeconds": 1
}
}],
"labels": {
"version": "v1",
"fluentd": "true"
},
"engineResources": {},
"replicas": 1,
"name": "nvidia-sp",
"svcOrchSpec": {
"resources": {
"requests": {
"cpu": "1"
}
}
},
"graph": {
"endpoint": {
"type": "REST"
},
"parameters": [{
"name": "url",
"type": "STRING",
"value": "localhost:2001"
},
{
"value": "spelling_model",
"name": "model_name",
"type": "STRING"
},
{
"name": "protocol",
"type": "STRING",
"value": "grpc"
}
],
"name": "sp-predictor",
"type": "MODEL",
"implementation": "UNKNOWN_IMPLEMENTATION"
}
},
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:07:04.066712118Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"textPayload": "predictor is now\n",
"insertId": "o5k4lnfh9xqje",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:07:04.066552975Z",
"severity": "INFO",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stdout",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:07:09.610592917Z"
},
{
"textPayload": "2019/08/28 18:06:49 http: TLS handshake error from 10.24.0.29:33778: tls: first record does not look like a TLS handshake\n",
"insertId": "1inw11of5u5v87",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:06:49.715414698Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:06:54.790364750Z"
},
{
"textPayload": "2019/08/28 18:06:34 http: TLS handshake error from 10.24.0.29:33584: tls: first record does not look like a TLS handshake\n",
"insertId": "1nxp38fg13pvl3j",
"resource": {
"type": "container",
"labels": {
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster"
}
},
"timestamp": "2019-08-28T18:06:34.715570710Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:06:39.690919380Z"
},
{
"textPayload": "2019/08/28 18:06:19 http: TLS handshake error from 10.24.0.29:33386: tls: first record does not look like a TLS handshake\n",
"insertId": "1dy51klfwqfv2s",
"resource": {
"type": "container",
"labels": {
"instance_id": "7174115594500519501",
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system"
}
},
"timestamp": "2019-08-28T18:06:19.715736777Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:06:22.637794442Z"
},
{
"textPayload": "2019/08/28 18:06:04 http: TLS handshake error from 10.24.0.29:33178: tls: first record does not look like a TLS handshake\n",
"insertId": "1t7qz7rg4pumrkh",
"resource": {
"type": "container",
"labels": {
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501",
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c"
}
},
"timestamp": "2019-08-28T18:06:04.715623845Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:06:11.015803586Z"
},
{
"textPayload": "2019/08/28 18:05:49 http: TLS handshake error from 10.24.0.29:32992: tls: first record does not look like a TLS handshake\n",
"insertId": "32ci4cg4odj5wj",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:05:49.715308334Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:05:56.016816047Z"
},
{
"textPayload": "2019/08/28 18:05:34 http: TLS handshake error from 10.24.0.29:32800: tls: first record does not look like a TLS handshake\n",
"insertId": "1e8b8blg4poppjf",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:05:34.715548553Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:05:40.919809308Z"
},
{
"textPayload": "2019/08/28 18:05:19 http: TLS handshake error from 10.24.0.29:60844: tls: first record does not look like a TLS handshake\n",
"insertId": "ezgleaf8edq6m",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:05:19.715306959Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:05:22.868793Z"
},
{
"textPayload": "2019/08/28 18:05:04 http: TLS handshake error from 10.24.0.29:60636: tls: first record does not look like a TLS handshake\n",
"insertId": "gxcht2fua6g00",
"resource": {
"type": "container",
"labels": {
"pod_id": "seldon-operator-controller-manager-0",
"zone": "us-east1-c",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:05:04.715344862Z",
"severity": "ERROR",
"labels": {
"container.googleapis.com/namespace_name": "seldon-system",
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:05:10.829873620Z"
},
{
"textPayload": "2019/08/28 18:04:49 http: TLS handshake error from 10.24.0.29:60450: tls: first record does not look like a TLS handshake\n",
"insertId": "1t7tl5ng4oh32q7",
"resource": {
"type": "container",
"labels": {
"zone": "us-east1-c",
"pod_id": "seldon-operator-controller-manager-0",
"project_id": "gn-data-science-project02",
"cluster_name": "temp-cluster",
"container_name": "manager",
"namespace_id": "seldon-system",
"instance_id": "7174115594500519501"
}
},
"timestamp": "2019-08-28T18:04:49.715417932Z",
"severity": "ERROR",
"labels": {
"compute.googleapis.com/resource_name": "gke-temp-cluster-default-pool-681bcb2d-tdnz",
"container.googleapis.com/pod_name": "seldon-operator-controller-manager-0",
"container.googleapis.com/stream": "stderr",
"container.googleapis.com/namespace_name": "seldon-system"
},
"logName": "projects/gn-data-science-project02/logs/manager",
"receiveTimestamp": "2019-08-28T18:04:54.740986395Z"
}
]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment