From ced955487eba5e37ba0323b64a5f6b7182e0c53d Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 16 Apr 2024 19:33:20 +0530 Subject: [PATCH 01/21] [helm] add nodeSelector --- .../templates/backend/backend-statefulset.yaml | 5 +++-- .../templates/frontend/frontend-deployment.yaml | 3 +++ .../syft/templates/mongo/mongo-statefulset.yaml | 3 +++ .../syft/templates/proxy/proxy-deployment.yaml | 3 +++ .../registry/registry-statefulset.yaml | 3 +++ .../seaweedfs/seaweedfs-statefulset.yaml | 5 +++-- packages/grid/helm/syft/values.yaml | 17 +++++++++++++++-- 7 files changed, 33 insertions(+), 6 deletions(-) diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index ce7a0051fd1..b00a35a7bd3 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -21,6 +21,9 @@ spec: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: backend spec: + {{- if .Values.node.nodeSelector }} + nodeSelector: {{- .Values.node.nodeSelector | toYaml | nindent 8 }} + {{- end }} containers: - name: backend-container image: {{ .Values.global.registry }}/openmined/grid-backend:{{ .Values.global.version }} @@ -104,8 +107,6 @@ spec: secretKeyRef: name: {{ .Values.seaweedfs.secretKeyName | required "seaweedfs.secretKeyName is required" }} key: s3RootPassword - - name: SEAWEED_MOUNT_PORT - value: {{ .Values.seaweedfs.mountApi.port | quote }} {{- end }} # Tracing - name: TRACE diff --git a/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml index 4147251b111..ba5c6e031b5 100644 --- a/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml @@ -19,6 +19,9 @@ spec: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: frontend spec: + {{- if .Values.frontend.nodeSelector }} + nodeSelector: {{- .Values.frontend.nodeSelector | toYaml | nindent 8 }} + {{- end }} containers: - name: frontend-container image: {{ .Values.global.registry }}/openmined/grid-frontend:{{ .Values.global.version }} diff --git a/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml index 6343aac499f..93824e1c2d4 100644 --- a/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml @@ -21,6 +21,9 @@ spec: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: mongo spec: + {{- if .Values.mongo.nodeSelector }} + nodeSelector: {{- .Values.mongo.nodeSelector | toYaml | nindent 8 }} + {{- end }} containers: - name: mongo-container image: mongo:7 diff --git a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml index 148cb2be29e..431d8372968 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml @@ -19,6 +19,9 @@ spec: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: proxy spec: + {{- if .Values.proxy.nodeSelector }} + nodeSelector: {{- .Values.proxy.nodeSelector | toYaml | nindent 8 }} + {{- end }} containers: - name: proxy-container image: traefik:v2.11.0 diff --git a/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml b/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml index 1e9366812d2..56fd34da3a1 100644 --- a/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml +++ b/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml @@ -20,6 +20,9 @@ spec: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: registry spec: + {{- if .Values.registry.nodeSelector }} + nodeSelector: {{- .Values.registry.nodeSelector | toYaml | nindent 8 }} + {{- end }} containers: - name: registry-container image: registry:2 diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml index fc151fcc2b3..aa156af2856 100644 --- a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml @@ -22,6 +22,9 @@ spec: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: seaweedfs spec: + {{- if .Values.seaweedfs.nodeSelector }} + nodeSelector: {{- .Values.seaweedfs.nodeSelector | toYaml | nindent 8 }} + {{- end }} containers: - name: seaweedfs-container image: {{ .Values.global.registry }}/openmined/grid-seaweedfs:{{ .Values.global.version }} @@ -37,8 +40,6 @@ spec: secretKeyRef: name: {{ .Values.seaweedfs.secretKeyName | required "seaweedfs.secretKeyName is required" }} key: s3RootPassword - - name: MOUNT_API_PORT - value: {{ .Values.seaweedfs.mountApi.port | quote }} {{- if .Values.seaweedfs.env }} {{- toYaml .Values.seaweedfs.env | nindent 12 }} {{- end }} diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 7e19227da36..4e4482b879c 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -14,6 +14,9 @@ mongo: # Extra environment vars env: null + # Node selector for pods + nodeSelector: null + # Pod Resource Limits resourcesPreset: nano resources: null @@ -55,8 +58,6 @@ seaweedfs: # Mount API mountApi: - port: 4001 - # automount: # - local_bucket: test-bucket # remote_bucket: @@ -76,6 +77,9 @@ seaweedfs: # - s3RootPassword secretKeyName: seaweedfs-secret + # Node selector for pods + nodeSelector: null + # Pod Resource Limits resourcesPreset: medium resources: null @@ -93,6 +97,9 @@ proxy: # Extra environment vars env: null + # Node selector for pods + nodeSelector: null + # Pod Resource Limits resourcesPreset: nano resources: null @@ -104,6 +111,9 @@ registry: # PVC storage size storageSize: 10Gi + # Node selector for pods + nodeSelector: null + # Pod Resource Limits resourcesPreset: nano resources: null @@ -132,6 +142,9 @@ node: # Extra environment vars env: null + # Node selector for pods + nodeSelector: null + # Pod Resource Limits resourcesPreset: medium resources: null From b70203708d34d0bb7a9803256fb0b71ee43ecbf1 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 17 Apr 2024 01:36:48 +0530 Subject: [PATCH 02/21] [helm] add podLabels & podAnnotations --- .../backend/backend-statefulset.yaml | 8 +++++++- .../frontend/frontend-deployment.yaml | 6 ++++++ .../templates/mongo/mongo-statefulset.yaml | 6 ++++++ .../templates/proxy/proxy-deployment.yaml | 6 ++++++ .../registry/registry-statefulset.yaml | 6 ++++++ .../seaweedfs/seaweedfs-statefulset.yaml | 6 ++++++ packages/grid/helm/syft/values.yaml | 20 +++++++++++++++++++ 7 files changed, 57 insertions(+), 1 deletion(-) diff --git a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml index b00a35a7bd3..8c9100f6d6b 100644 --- a/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend/backend-statefulset.yaml @@ -20,6 +20,12 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: backend + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.node.podAnnotations }} + annotations: {{- toYaml .Values.node.podAnnotations | nindent 8 }} + {{- end }} spec: {{- if .Values.node.nodeSelector }} nodeSelector: {{- .Values.node.nodeSelector | toYaml | nindent 8 }} @@ -45,7 +51,7 @@ spec: - name: NODE_TYPE value: {{ .Values.node.type | default "domain" | quote }} - name: NODE_NAME - value: {{ .Values.node.name | default .Release.Name | quote }} + value: {{ .Values.node.name | default (printf "%s-node" .Release.Name) | quote }} - name: NODE_SIDE_TYPE value: {{ .Values.node.side | quote }} - name: DEFAULT_ROOT_EMAIL diff --git a/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml index ba5c6e031b5..58d1c4d2f7b 100644 --- a/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend/frontend-deployment.yaml @@ -18,6 +18,12 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: frontend + {{- if .Values.frontend.podLabels }} + {{- toYaml .Values.frontend.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.frontend.podAnnotations }} + annotations: {{- toYaml .Values.frontend.podAnnotations | nindent 8 }} + {{- end }} spec: {{- if .Values.frontend.nodeSelector }} nodeSelector: {{- .Values.frontend.nodeSelector | toYaml | nindent 8 }} diff --git a/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml index 93824e1c2d4..91060b90a9b 100644 --- a/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo/mongo-statefulset.yaml @@ -20,6 +20,12 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: mongo + {{- if .Values.mongo.podLabels }} + {{- toYaml .Values.mongo.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.mongo.podAnnotations }} + annotations: {{- toYaml .Values.mongo.podAnnotations | nindent 8 }} + {{- end }} spec: {{- if .Values.mongo.nodeSelector }} nodeSelector: {{- .Values.mongo.nodeSelector | toYaml | nindent 8 }} diff --git a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml index 431d8372968..6adb42f6c9c 100644 --- a/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml +++ b/packages/grid/helm/syft/templates/proxy/proxy-deployment.yaml @@ -18,6 +18,12 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: proxy + {{- if .Values.proxy.podLabels }} + {{- toYaml .Values.proxy.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.proxy.podAnnotations }} + annotations: {{- toYaml .Values.proxy.podAnnotations | nindent 8 }} + {{- end }} spec: {{- if .Values.proxy.nodeSelector }} nodeSelector: {{- .Values.proxy.nodeSelector | toYaml | nindent 8 }} diff --git a/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml b/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml index 56fd34da3a1..baefc45dce0 100644 --- a/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml +++ b/packages/grid/helm/syft/templates/registry/registry-statefulset.yaml @@ -19,6 +19,12 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: registry + {{- if .Values.registry.podLabels }} + {{- toYaml .Values.registry.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.registry.podAnnotations }} + annotations: {{- toYaml .Values.registry.podAnnotations | nindent 8 }} + {{- end }} spec: {{- if .Values.registry.nodeSelector }} nodeSelector: {{- .Values.registry.nodeSelector | toYaml | nindent 8 }} diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml index aa156af2856..1d1dd8cae21 100644 --- a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml @@ -21,6 +21,12 @@ spec: labels: {{- include "common.labels" . | nindent 8 }} app.kubernetes.io/component: seaweedfs + {{- if .Values.seaweedfs.podLabels }} + {{- toYaml .Values.seaweedfs.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.seaweedfs.podAnnotations }} + annotations: {{- toYaml .Values.seaweedfs.podAnnotations | nindent 8 }} + {{- end }} spec: {{- if .Values.seaweedfs.nodeSelector }} nodeSelector: {{- .Values.seaweedfs.nodeSelector | toYaml | nindent 8 }} diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 4e4482b879c..843e0242bf3 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -14,6 +14,10 @@ mongo: # Extra environment vars env: null + # Pod labels & annotations + podLabels: null + podAnnotations: null + # Node selector for pods nodeSelector: null @@ -77,6 +81,10 @@ seaweedfs: # - s3RootPassword secretKeyName: seaweedfs-secret + # Pod labels & annotations + podLabels: null + podAnnotations: null + # Node selector for pods nodeSelector: null @@ -97,6 +105,10 @@ proxy: # Extra environment vars env: null + # Pod labels & annotations + podLabels: null + podAnnotations: null + # Node selector for pods nodeSelector: null @@ -111,6 +123,10 @@ registry: # PVC storage size storageSize: 10Gi + # Pod labels & annotations + podLabels: null + podAnnotations: null + # Node selector for pods nodeSelector: null @@ -142,6 +158,10 @@ node: # Extra environment vars env: null + # Pod labels & annotations + podLabels: null + podAnnotations: null + # Node selector for pods nodeSelector: null From c417c45849953a379b4c4e635e7cfad0ff0dd9cb Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 17 Apr 2024 01:47:29 +0530 Subject: [PATCH 03/21] [helm] update resource limits --- packages/grid/helm/syft/values.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/grid/helm/syft/values.yaml b/packages/grid/helm/syft/values.yaml index 843e0242bf3..6b953dd9aa8 100644 --- a/packages/grid/helm/syft/values.yaml +++ b/packages/grid/helm/syft/values.yaml @@ -22,7 +22,7 @@ mongo: nodeSelector: null # Pod Resource Limits - resourcesPreset: nano + resourcesPreset: large resources: null # PVC storage size @@ -40,7 +40,7 @@ frontend: env: null # Pod Resource Limits - resourcesPreset: micro + resourcesPreset: medium resources: null seaweedfs: @@ -74,7 +74,7 @@ seaweedfs: env: null # PVC storage size - storageSize: 10Gi + storageSize: 100Gi # Seaweed secret name. Override this if you want to use a self-managed secret. # Secret must contain the following keys: @@ -89,7 +89,7 @@ seaweedfs: nodeSelector: null # Pod Resource Limits - resourcesPreset: medium + resourcesPreset: xlarge resources: null # extra volume mounts @@ -113,7 +113,7 @@ proxy: nodeSelector: null # Pod Resource Limits - resourcesPreset: nano + resourcesPreset: small resources: null registry: @@ -121,7 +121,7 @@ registry: env: null # PVC storage size - storageSize: 10Gi + storageSize: 50Gi # Pod labels & annotations podLabels: null @@ -131,7 +131,7 @@ registry: nodeSelector: null # Pod Resource Limits - resourcesPreset: nano + resourcesPreset: medium resources: null node: @@ -166,7 +166,7 @@ node: nodeSelector: null # Pod Resource Limits - resourcesPreset: medium + resourcesPreset: xlarge resources: null # Seaweed secret name. Override this if you want to use a self-managed secret. From a8154f84250a1b2bbc63b69f3afe9429b4c341a2 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 17 Apr 2024 01:55:07 +0530 Subject: [PATCH 04/21] [helm] add profiles to devspace --- .gitignore | 3 +- packages/grid/devspace.yaml | 31 ++++++- .../grid/helm/examples/azure/azure.high.yaml | 34 ++++++++ packages/grid/helm/examples/gcp/README.md | 41 ++++++++++ packages/grid/helm/examples/gcp/gcp.high.yaml | 81 ++++++++++++++++++ packages/grid/helm/examples/gcp/gcp.low.yaml | 82 +++++++++++++++++++ .../grid/helm/examples/gcp/gcp.nosync.yaml | 47 +++++++++++ 7 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 packages/grid/helm/examples/azure/azure.high.yaml create mode 100644 packages/grid/helm/examples/gcp/README.md create mode 100644 packages/grid/helm/examples/gcp/gcp.high.yaml create mode 100644 packages/grid/helm/examples/gcp/gcp.low.yaml create mode 100644 packages/grid/helm/examples/gcp/gcp.nosync.yaml diff --git a/.gitignore b/.gitignore index 33dc85c251c..efb1fb82edd 100644 --- a/.gitignore +++ b/.gitignore @@ -73,5 +73,6 @@ nohup.out # notebook data notebooks/helm/scenario_data.jsonl + # tox syft.build.helm generated file -out.txt +out.* diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 93d4922532f..c121fabcc29 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -22,9 +22,10 @@ pipelines: create_deployments --all vars: - DEVSPACE_ENV_FILE: "default.env" + DOCKER_IMAGE_BACKEND: openmined/grid-backend + DOCKER_IMAGE_FRONTEND: openmined/grid-frontend + DOCKER_IMAGE_SEAWEEDFS: openmined/grid-seaweedfs CONTAINER_REGISTRY: "docker.io" - NODE_NAME: "mynode" VERSION: "0.8.7-beta.1" PLATFORM: $(uname -m | grep -q 'arm64' && echo "arm64" || echo "amd64") @@ -38,6 +39,7 @@ images: context: ../ tags: - dev-${DEVSPACE_TIMESTAMP} + - dev frontend: image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}" buildKit: @@ -48,6 +50,7 @@ images: context: ./frontend tags: - dev-${DEVSPACE_TIMESTAMP} + - dev seaweedfs: image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}" buildKit: {} @@ -55,6 +58,7 @@ images: context: ./seaweedfs tags: - dev-${DEVSPACE_TIMESTAMP} + - dev # This is a list of `deployments` that DevSpace can create for this project deployments: @@ -67,8 +71,6 @@ deployments: global: registry: ${CONTAINER_REGISTRY} version: dev-${DEVSPACE_TIMESTAMP} - node: - name: ${NODE_NAME} # anything that does not need devspace $env vars should go in values.dev.yaml valuesFiles: - ./helm/values.dev.yaml @@ -117,3 +119,24 @@ profiles: path: images.seaweedfs - op: remove path: dev.seaweedfs + + - name: gcp + patches: + - op: replace + path: deployments.syft.helm.valuesFiles + value: + - ./helm/examples/gcp/gcp.high.yaml + + - name: gcp-low + patches: + - op: replace + path: deployments.syft.helm.valuesFiles + value: + - ./helm/examples/gcp/gcp.low.yaml + + - name: azure + patches: + - op: replace + path: deployments.syft.helm.valuesFiles + value: + - ./helm/examples/azure/azure.high.yaml diff --git a/packages/grid/helm/examples/azure/azure.high.yaml b/packages/grid/helm/examples/azure/azure.high.yaml new file mode 100644 index 00000000000..4f45b193846 --- /dev/null +++ b/packages/grid/helm/examples/azure/azure.high.yaml @@ -0,0 +1,34 @@ +# =================================================== +# Deploy Syft High Side Node to an AKS Cluster +# DOES NOT sync seaweedfs to a Azure bucket +# =================================================== + +node: + name: syft-azure + side: high + rootEmail: info@openmined.org + defaultWorkerPoolCount: 1 + resourcesPreset: 2xlarge + +ingress: + # Make sure cluster is created with --enable-app-routing + # az aks create -g group-name -n cluster-name -l region --enable-app-routing + className: webapprouting.kubernetes.azure.com + # or, has azure-application-gateway setup through dashboard or CLI + # className: azure-application-gateway + + +seaweedfs: + resourcesPreset: 2xlarge + +proxy: + resourcesPreset: medium + +registry: + resourcesPreset: medium + +frontend: + resourcesPreset: medium + +mongo: + resourcesPreset: large diff --git a/packages/grid/helm/examples/gcp/README.md b/packages/grid/helm/examples/gcp/README.md new file mode 100644 index 00000000000..250a2338771 --- /dev/null +++ b/packages/grid/helm/examples/gcp/README.md @@ -0,0 +1,41 @@ +# GCS Deployment + +## Resource Links +* [Autopilot Overview](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview) +* [Autopilot Resource Limits Defaults](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#compute-class-defaults) +* [AutoPilot Resource Limits Min/Max](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-resource-requests#min-max-requests) +* [Compute Classes](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-compute-classes) +* [Performance Pods](https://cloud.google.com/kubernetes-engine/docs/how-to/performance-pods) + +## Setup + +Helm `values.yaml` for high & low side deployments +- [`gcp.high.yaml`](./gcp.high.yaml) +- [`gcp.low.yaml`](./gcp.low.yaml) + +Deployment on GKE with SeaweedFS sync to GCS requires: +1. A GCS bucket in the same project where the cluster will be deployed + * `syft-bucket-high` (for high side deployment) + * `syft-bucket-low` (for low side deployment) + +2. An IAM service account with sufficient permissions to read/write/delete object to these buckets + +## For Autopilot Cluster + +Uncomment the `nodeSelector` to use a specific compute class + machine family. + +Set resource limits in-line with the machine family + +```yaml + nodeSelector: + cloud.google.com/compute-class: Performance + cloud.google.com/machine-family: c3 + + resources: + requests: + cpu: 2 + memory: "8Gi" + limits: + cpu: 4 + memory: "16Gi" +``` diff --git a/packages/grid/helm/examples/gcp/gcp.high.yaml b/packages/grid/helm/examples/gcp/gcp.high.yaml new file mode 100644 index 00000000000..338fb001cb2 --- /dev/null +++ b/packages/grid/helm/examples/gcp/gcp.high.yaml @@ -0,0 +1,81 @@ +# =================================================== +# Deploy Syft High Side Node to a GKE Cluster +# Syncs seaweedfs to a GCS bucket "syft-bucket-high" +# =================================================== + +node: + name: syft-gcp + side: high + + # Force backend to write results to this bucket + # should be same as mountApi.mounts.local_bucket + defaultBucketName: syft-bucket-high-gcs + + # For autopilot clusters with GKE 1.28+, uncomment this + # nodeSelector: + # cloud.google.com/compute-class: Performance + # cloud.google.com/machine-family: c3 + + # Resources set inline with c3-standard-4 machine type + resources: + requests: + cpu: 2 + memory: "8Gi" + limits: + cpu: 4 + memory: "16Gi" + +seaweedfs: + storageSize: 100Gi + + mountApi: + # Mount remote GCS bucket named "syft-bucket-high" to seaweedfs bucket named "syft-bucket-high-gcs + mounts: + - local_bucket: syft-bucket-high-gcs + remote_bucket: + type: gcs + bucket_name: syft-bucket-high + # filename should be as secret.mountCredentials.name + # directory will always be /run/secrets/ + creds: /run/secrets/gcs_creds.json + + secret: + mountCredentials: + # remove this comment and paste the Service Account JSON Here + # TODO: move to workload identity + - name: gcs_creds.json + value: | + { + "": "" + } + + # For autopilot clusters with GKE 1.28+, uncomment this + # nodeSelector: + # cloud.google.com/compute-class: Performance + # cloud.google.com/machine-family: c3 + + # Resources set inline with c3-standard-4 machine type + resources: + requests: + cpu: 2 + memory: "8Gi" + limits: + cpu: 4 + memory: "16Gi" + +ingress: + # GKE 1.28+ need ingressClassName + className: gce + class: gce + +proxy: + resourcesPreset: medium + +registry: + resourcesPreset: medium + +frontend: + resourcesPreset: medium + +mongo: + resourcesPreset: large diff --git a/packages/grid/helm/examples/gcp/gcp.low.yaml b/packages/grid/helm/examples/gcp/gcp.low.yaml new file mode 100644 index 00000000000..8300bc848c6 --- /dev/null +++ b/packages/grid/helm/examples/gcp/gcp.low.yaml @@ -0,0 +1,82 @@ +# =================================================== +# Deploy Syft Low Side Node to a GKE Cluster +# Syncs seaweedfs to a GCS bucket "syft-bucket-low" +# =================================================== + +node: + name: syft-gcp + side: low + + # Force backend to write results to this bucket + # should be same as mountApi.mounts.local_bucket + defaultBucketName: syft-bucket-low-gcs + + # For autopilot clusters with GKE 1.28+, uncomment this + # nodeSelector: + # cloud.google.com/compute-class: Performance + # cloud.google.com/machine-family: c3 + + # Resources set inline with c3-standard-4 machine type + resources: + requests: + cpu: 2 + memory: "8Gi" + limits: + cpu: 4 + memory: "16Gi" + +seaweedfs: + storageSize: 100Gi + + mountApi: + # Mount remote GCS bucket named "syft-bucket-low" to seaweedfs bucket named "syft-bucket-low-gcs + mounts: + - local_bucket: syft-bucket-low-gcs + remote_bucket: + type: gcs + bucket_name: syft-bucket-low + # filename should be as secret.mountCredentials.name + # directory will always be /run/secrets/ + creds: /run/secrets/gcs_creds.json + + secret: + mountCredentials: + # remove this comment and paste the Service Account JSON Here + # TODO: move to workload identity + - name: gcs_creds.json + value: | + { + "": "" + } + + # For autopilot clusters with GKE 1.28+, uncomment this + # nodeSelector: + # cloud.google.com/compute-class: Performance + # cloud.google.com/machine-family: c3 + + # Resources set inline with c3-standard-4 machine type + resources: + requests: + cpu: 2 + memory: "8Gi" + limits: + cpu: 4 + memory: "16Gi" + +ingress: + # GKE 1.28+ need ingressClassName + className: gce + class: gce + +proxy: + resourcesPreset: medium + +registry: + resourcesPreset: medium + storageSize: 50Gi + +frontend: + resourcesPreset: medium + +mongo: + resourcesPreset: large diff --git a/packages/grid/helm/examples/gcp/gcp.nosync.yaml b/packages/grid/helm/examples/gcp/gcp.nosync.yaml new file mode 100644 index 00000000000..0276e67277d --- /dev/null +++ b/packages/grid/helm/examples/gcp/gcp.nosync.yaml @@ -0,0 +1,47 @@ +# =================================================== +# Deploy Syft High Side Node to a GKE Cluster +# DOES NOT sync seaweedfs to a GCS bucket +# =================================================== + +node: + name: syft-gcp + side: high + + # Resources set inline with c3-standard-4 machine type + resources: + requests: + cpu: 2 + memory: "8Gi" + limits: + cpu: 4 + memory: "16Gi" + +seaweedfs: + storageSize: 100Gi + + # Resources set inline with c3-standard-4 machine type + resources: + requests: + cpu: 2 + memory: "8Gi" + limits: + cpu: 4 + memory: "16Gi" + +ingress: + # GKE 1.28+ need ingressClassName + className: gce + class: gce + +proxy: + resourcesPreset: medium + +registry: + resourcesPreset: medium + storageSize: 100Gi + +frontend: + resourcesPreset: medium + +mongo: + resourcesPreset: large From a7efc49eb963a51325d7b718345ced035b0595d1 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 17 Apr 2024 12:24:34 +0530 Subject: [PATCH 05/21] [devspace] remove dev tag --- packages/grid/devspace.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index c121fabcc29..013f9e6c48b 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -39,7 +39,6 @@ images: context: ../ tags: - dev-${DEVSPACE_TIMESTAMP} - - dev frontend: image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_FRONTEND}" buildKit: @@ -50,7 +49,6 @@ images: context: ./frontend tags: - dev-${DEVSPACE_TIMESTAMP} - - dev seaweedfs: image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_SEAWEEDFS}" buildKit: {} @@ -58,7 +56,6 @@ images: context: ./seaweedfs tags: - dev-${DEVSPACE_TIMESTAMP} - - dev # This is a list of `deployments` that DevSpace can create for this project deployments: From 34e7e7c22e2bdf2066148d0b1b7bbb69af45eb6a Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 17 Apr 2024 10:05:49 +0200 Subject: [PATCH 06/21] setup a failing test for an existing bug --- .gitignore | 3 ++ .../syft/src/syft/service/sync/diff_state.py | 1 - .../syft/service/sync/sync_widget_test.py | 50 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 packages/syft/tests/syft/service/sync/sync_widget_test.py diff --git a/.gitignore b/.gitignore index 33dc85c251c..3c70cadfc88 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,6 @@ nohup.out notebooks/helm/scenario_data.jsonl # tox syft.build.helm generated file out.txt + +# gitconfig +.git-blame-ignore-revs \ No newline at end of file diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index 7ad7973573e..e735c5ea01d 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1232,7 +1232,6 @@ def from_widget_state( ) ] - # mockify mockify = widget.mockify # storage permissions diff --git a/packages/syft/tests/syft/service/sync/sync_widget_test.py b/packages/syft/tests/syft/service/sync/sync_widget_test.py new file mode 100644 index 00000000000..5c6801d92c1 --- /dev/null +++ b/packages/syft/tests/syft/service/sync/sync_widget_test.py @@ -0,0 +1,50 @@ +# stdlib + +# syft absolute +import syft +from syft.client.domain_client import DomainClient +from syft.client.syncing import compare_clients +from syft.client.syncing import resolve_single +from syft.service.job.job_stash import Job + + +def compare_and_resolve(*, from_client: DomainClient, to_client: DomainClient): + diff_state_before = compare_clients(from_client, to_client) + for obj_diff_batch in diff_state_before.batches: + widget = resolve_single(obj_diff_batch) + widget.click_sync() + from_client.refresh() + to_client.refresh() + diff_state_after = compare_clients(from_client, to_client) + return diff_state_before, diff_state_after + + +def run_and_accept_result(client): + job_high = client.code.compute(blocking=True) + client.requests[0].accept_by_depositing_result(job_high) + return job_high + + +@syft.syft_function_single_use() +def compute() -> int: + return 42 + + +def test_diff_widget_merge_status(low_worker, high_worker): + low_client = low_worker.root_client + client_low_ds = low_worker.guest_client + high_client = high_worker.root_client + + _ = client_low_ds.code.request_code_execution(compute) + + diff_before, diff_after = compare_and_resolve( + from_client=low_client, to_client=high_client + ) + run_and_accept_result(high_client) + diff_before, diff_after = compare_and_resolve( + from_client=high_client, to_client=low_client + ) + + # expect to have a job diff, might fail if the implementation changes + job_diff = [batch for batch in diff_after.batches if batch.root_type is Job][0] + assert job_diff.status == "SAME", "Job diff status should be SAME after syncing" From a568fb5488709c6c59926a520549cb77ee03cc38 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 17 Apr 2024 13:50:54 +0200 Subject: [PATCH 07/21] add docstring --- packages/syft/src/syft/service/action/action_object.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index 5b0de22410e..f77d7a8330e 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -2059,6 +2059,11 @@ def __rrshift__(self, other: Any) -> Any: @serializable() class AnyActionObject(ActionObject): + """ + This is a catch-all class for all objects that are not + defined in the `action_types` dictionary. + """ + __canonical_name__ = "AnyActionObject" __version__ = SYFT_OBJECT_VERSION_3 From 47e23b52ba9e8cdf5b95242aab36831a46fe3dfb Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 17 Apr 2024 13:59:01 +0200 Subject: [PATCH 08/21] simplify action_type_for_type --- packages/syft/src/syft/service/action/action_types.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_types.py b/packages/syft/src/syft/service/action/action_types.py index a1db49f8a59..9721a48ec8e 100644 --- a/packages/syft/src/syft/service/action/action_types.py +++ b/packages/syft/src/syft/service/action/action_types.py @@ -15,17 +15,15 @@ def action_type_for_type(obj_or_type: Any) -> type: obj_or_type: Union[object, type] Can be an object or a class """ + if isinstance(obj_or_type, ActionDataEmpty): + obj_or_type = obj_or_type.syft_internal_type if type(obj_or_type) != type: - if isinstance(obj_or_type, ActionDataEmpty): - obj_or_type = obj_or_type.syft_internal_type - else: - obj_or_type = type(obj_or_type) + obj_or_type = type(obj_or_type) if obj_or_type not in action_types: debug(f"WARNING: No Type for {obj_or_type}, returning {action_types[Any]}") - return action_types[Any] - return action_types[obj_or_type] + return action_types.get(obj_or_type, action_types[Any]) def action_type_for_object(obj: Any) -> type: From 4d5deca8b362d38a61a04000b433f82068eeb088 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 17 Apr 2024 18:21:10 +0530 Subject: [PATCH 09/21] fix insufficient CPU in CI --- packages/grid/helm/values.dev.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/grid/helm/values.dev.yaml b/packages/grid/helm/values.dev.yaml index 06573cee813..40415f5b3ea 100644 --- a/packages/grid/helm/values.dev.yaml +++ b/packages/grid/helm/values.dev.yaml @@ -5,20 +5,39 @@ global: randomizedSecrets: false registry: + resourcesPreset: null + resources: null + storageSize: "5Gi" node: rootEmail: info@openmined.org defaultWorkerPoolCount: 1 + resourcesPreset: null + resources: null + secret: defaultRootPassword: changethis mongo: + resourcesPreset: null + resources: null + secret: rootPassword: example seaweedfs: + resourcesPreset: null + resources: null + secret: s3RootPassword: admin +frontend: + resourcesPreset: null + resources: null + +proxy: + resourcesPreset: null + resources: null From 63bde71611b995580e7accb98ede75b651bd496e Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Wed, 17 Apr 2024 15:45:12 +0200 Subject: [PATCH 10/21] wip --- packages/syft/src/syft/service/action/action_object.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index f77d7a8330e..f03afdd2686 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -2078,6 +2078,11 @@ def __float__(self) -> float: def __int__(self) -> float: return int(self.syft_action_data) + def syft_eq(self, ext_obj: Self | None) -> bool: + if ext_obj is None: + return False + return self.id.id == ext_obj.id.id + action_types[Any] = AnyActionObject From c06911aa5bd185c67e7370382c67bb7401880f65 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 17 Apr 2024 21:16:42 +0530 Subject: [PATCH 11/21] fix wait_for seaweedfs --- .../seaweedfs/seaweedfs-statefulset.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml index 1d1dd8cae21..dd882841ab0 100644 --- a/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs/seaweedfs-statefulset.yaml @@ -49,6 +49,25 @@ spec: {{- if .Values.seaweedfs.env }} {{- toYaml .Values.seaweedfs.env | nindent 12 }} {{- end }} + ports: + - name: mount-api-port + containerPort: 4001 + - name: master-port + containerPort: 9333 + startupProbe: + httpGet: + path: /?probe=startupProbe + port: mount-api-port + failureThreshold: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /cluster/healthz?probe=livenessProbe + port: master-port + periodSeconds: 60 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: null volumeMounts: # master + filer + volume data - mountPath: /data/ From 78d5477dc9c4fa48c646ddfae11bbee29b644532 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 17 Apr 2024 21:47:19 +0530 Subject: [PATCH 12/21] fix incorrect node type --- packages/grid/devspace.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 013f9e6c48b..473a9d82e24 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -68,6 +68,8 @@ deployments: global: registry: ${CONTAINER_REGISTRY} version: dev-${DEVSPACE_TIMESTAMP} + node: + type: domain # required for the gateway profile # anything that does not need devspace $env vars should go in values.dev.yaml valuesFiles: - ./helm/values.dev.yaml From c6e2a52b412318286bf82860bd1d6662fc915896 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Thu, 18 Apr 2024 08:23:20 +0200 Subject: [PATCH 13/21] add warning print --- packages/syft/src/syft/service/sync/diff_state.py | 2 ++ packages/syft/src/syft/service/sync/resolve_widget.py | 11 ++++++++++- .../syft/tests/syft/service/sync/sync_widget_test.py | 3 +-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index e735c5ea01d..457ee5f2633 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -1233,6 +1233,8 @@ def from_widget_state( ] mockify = widget.mockify + if widget.has_unused_share_button: + print("Share button was not used, so we will mockify the object") # storage permissions new_storage_permissions = [] diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 10083541917..7355b8893ec 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -120,6 +120,11 @@ def set_share_private_data(self) -> None: def mockify(self) -> bool: return not self.share_private_data + @property + def has_unused_share_button(self) -> bool: + # does not have share button + return False + @property def share_private_data(self) -> bool: # there are TwinAPIEndpoint.__private_sync_attr_mocks__ @@ -191,11 +196,15 @@ def __init__( def mockify(self) -> bool: if isinstance(self.diff.non_empty_object, TwinAPIEndpoint): return True - if self.show_share_button and not self.share_private_data: + if self.has_unused_share_button: return True else: return False + @property + def has_unused_share_button(self) -> bool: + return self.show_share_button and not self.share_private_data + @property def show_share_button(self) -> bool: return isinstance(self.diff.non_empty_object, SyftLog | ActionObject) diff --git a/packages/syft/tests/syft/service/sync/sync_widget_test.py b/packages/syft/tests/syft/service/sync/sync_widget_test.py index 5c6801d92c1..81e0db4d41b 100644 --- a/packages/syft/tests/syft/service/sync/sync_widget_test.py +++ b/packages/syft/tests/syft/service/sync/sync_widget_test.py @@ -1,5 +1,3 @@ -# stdlib - # syft absolute import syft from syft.client.domain_client import DomainClient @@ -12,6 +10,7 @@ def compare_and_resolve(*, from_client: DomainClient, to_client: DomainClient): diff_state_before = compare_clients(from_client, to_client) for obj_diff_batch in diff_state_before.batches: widget = resolve_single(obj_diff_batch) + widget.click_share_all_private_data() widget.click_sync() from_client.refresh() to_client.refresh() From e20ada92f0e81ff2232fe8ceb9815cba7c342578 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Thu, 18 Apr 2024 10:57:18 +0200 Subject: [PATCH 14/21] revert setup --- packages/syft/setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 6c5b048631b..6147adce6d0 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -68,7 +68,6 @@ syft = PyYAML==6.0.1 azure-storage-blob==12.19.1 ipywidgets==8.1.2 - google-cloud-bigquery[pandas] install_requires = %(syft)s From 68de569811775f6719b3ef425cefb318be743ede Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Thu, 18 Apr 2024 12:06:49 +0200 Subject: [PATCH 15/21] clarify error message for __getattribute__ --- packages/syft/src/syft/client/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/client/api.py b/packages/syft/src/syft/client/api.py index e07a6552550..65174e7ce62 100644 --- a/packages/syft/src/syft/client/api.py +++ b/packages/syft/src/syft/client/api.py @@ -573,7 +573,8 @@ def __getattribute__(self, name: str) -> Any: except AttributeError: raise SyftAttributeError( f"'APIModule' api{self.path} object has no submodule or method '{name}', " - "you may not have permission to access the module you are trying to access" + "you may not have permission to access the module you are trying to access." + "If you think this is an error, try calling `client.refresh()` to update the API." ) def __getitem__(self, key: str | int) -> Any: From 09aa6b4dc6ee63cd77d369ae2aa39b69010fde03 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Thu, 18 Apr 2024 12:08:33 +0200 Subject: [PATCH 16/21] fix is_execution_on_owned_args for empty kwargs --- packages/syft/src/syft/service/code/user_code_service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/syft/src/syft/service/code/user_code_service.py b/packages/syft/src/syft/service/code/user_code_service.py index e5c1e590d73..a8ef1687933 100644 --- a/packages/syft/src/syft/service/code/user_code_service.py +++ b/packages/syft/src/syft/service/code/user_code_service.py @@ -384,7 +384,9 @@ def keep_owned_kwargs( def is_execution_on_owned_args( self, kwargs: dict[str, Any], context: AuthedServiceContext ) -> bool: - return len(self.keep_owned_kwargs(kwargs, context)) == len(kwargs) + return bool(kwargs) and len(self.keep_owned_kwargs(kwargs, context)) == len( + kwargs + ) @service_method(path="code.call", name="call", roles=GUEST_ROLE_LEVEL) def call( From b7c2d197e93e49f4bcee2784825f8b1c38ae251c Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Thu, 18 Apr 2024 12:15:06 +0200 Subject: [PATCH 17/21] fix repr for high state --- .../syft/src/syft/service/sync/diff_state.py | 39 ++++++ .../src/syft/service/sync/resolve_widget.py | 41 +++--- .../tests/syft/service/sync/sync_flow_test.py | 118 ++++++++++++------ .../syft/service/sync/sync_widget_test.py | 92 +++++++++++++- 4 files changed, 229 insertions(+), 61 deletions(-) diff --git a/packages/syft/src/syft/service/sync/diff_state.py b/packages/syft/src/syft/service/sync/diff_state.py index 457ee5f2633..1dda52759b1 100644 --- a/packages/syft/src/syft/service/sync/diff_state.py +++ b/packages/syft/src/syft/service/sync/diff_state.py @@ -16,6 +16,7 @@ from typing_extensions import Self # relative +from ...client.api import APIRegistry from ...client.client import SyftClient from ...client.sync_decision import SyncDecision from ...client.sync_decision import SyncDirection @@ -576,6 +577,15 @@ def target_node_uid(self) -> UID: else: return self.low_node_uid + @property + def source_node_uid(self) -> UID: + if self.sync_direction is None: + raise ValueError("no direction specified") + if self.sync_direction == SyncDirection.LOW_TO_HIGH: + return self.low_node_uid + else: + return self.high_node_uid + @property def target_verify_key(self) -> SyftVerifyKey: if self.sync_direction is None: @@ -585,6 +595,35 @@ def target_verify_key(self) -> SyftVerifyKey: else: return self.user_verify_key_low + @property + def source_verify_key(self) -> SyftVerifyKey: + if self.sync_direction is None: + raise ValueError("no direction specified") + if self.sync_direction == SyncDirection.LOW_TO_HIGH: + return self.user_verify_key_low + else: + return self.user_verify_key_high + + @property + def source_client(self) -> SyftClient: + return self.build(self.source_node_uid, self.source_verify_key) + + @property + def target_client(self) -> SyftClient: + return self.build(self.target_node_uid, self.target_verify_key) + + def build(self, node_uid: UID, syft_client_verify_key: SyftVerifyKey): # type: ignore + # relative + from ...client.domain_client import DomainClient + + api = APIRegistry.api_for(node_uid, syft_client_verify_key) + client = DomainClient( + api=api, + connection=api.connection, # type: ignore + credentials=api.signing_key, # type: ignore + ) + return client + def get_dependencies( self, include_roots: bool = False, diff --git a/packages/syft/src/syft/service/sync/resolve_widget.py b/packages/syft/src/syft/service/sync/resolve_widget.py index 7355b8893ec..0add00738ec 100644 --- a/packages/syft/src/syft/service/sync/resolve_widget.py +++ b/packages/syft/src/syft/service/sync/resolve_widget.py @@ -15,7 +15,6 @@ from ipywidgets import VBox # relative -from ...client.api import APIRegistry from ...client.sync_decision import SyncDecision from ...client.sync_decision import SyncDirection from ...node.credentials import SyftVerifyKey @@ -426,8 +425,12 @@ def button_callback(self, *args: list, **kwargs: dict) -> SyftSuccess | SyftErro # Maybe default read permission for some objects (high -> low) # TODO: UID - resolved_state_low = ResolvedSyncState(node_uid=UID(), alias="low") - resolved_state_high = ResolvedSyncState(node_uid=UID(), alias="high") + resolved_state_low = ResolvedSyncState( + node_uid=self.obj_diff_batch.low_node_uid, alias="low" + ) + resolved_state_high = ResolvedSyncState( + node_uid=self.obj_diff_batch.high_node_uid, alias="high" + ) batch_diff = self.obj_diff_batch if batch_diff.is_unchanged: @@ -495,25 +498,23 @@ def button_callback(self, *args: list, **kwargs: dict) -> SyftSuccess | SyftErro resolved_state_low.add_sync_instruction(sync_instruction) resolved_state_high.add_sync_instruction(sync_instruction) - # TODO: ONLY WORKS FOR LOW TO HIGH - # relative - from ...client.domain_client import DomainClient - - api = APIRegistry.api_for( - self.obj_diff_batch.target_node_uid, self.obj_diff_batch.target_verify_key - ) - client = DomainClient( - api=api, - connection=api.connection, # type: ignore - credentials=api.signing_key, # type: ignore - ) - if self.obj_diff_batch.sync_direction is None: raise ValueError("no direction specified") - if self.obj_diff_batch.sync_direction == SyncDirection.LOW_TO_HIGH: - res = client.apply_state(resolved_state_high) - else: - res = client.apply_state(resolved_state_low) + sync_direction = self.obj_diff_batch.sync_direction + resolved_state = ( + resolved_state_high + if sync_direction == SyncDirection.LOW_TO_HIGH + else resolved_state_low + ) + res = self.obj_diff_batch.target_client.apply_state(resolved_state) + + if sync_direction == SyncDirection.HIGH_TO_LOW: + # apply empty state to generete a new state + resolved_state_high = ResolvedSyncState( + node_uid=self.obj_diff_batch.high_node_uid, alias="high" + ) + high_client = self.obj_diff_batch.source_client + res = high_client.apply_state(resolved_state_high) self.is_synced = True self.set_result_state(res) diff --git a/packages/syft/tests/syft/service/sync/sync_flow_test.py b/packages/syft/tests/syft/service/sync/sync_flow_test.py index e73a1b193a0..fae5fdfb18f 100644 --- a/packages/syft/tests/syft/service/sync/sync_flow_test.py +++ b/packages/syft/tests/syft/service/sync/sync_flow_test.py @@ -7,8 +7,10 @@ import pytest # syft absolute +import syft import syft as sy from syft.abstract_node import NodeSideType +from syft.client.domain_client import DomainClient from syft.client.sync_decision import SyncDecision from syft.client.syncing import compare_clients from syft.client.syncing import compare_states @@ -16,6 +18,41 @@ from syft.client.syncing import resolve_single from syft.service.action.action_object import ActionObject from syft.service.response import SyftError +from syft.service.response import SyftSuccess + + +def compare_and_resolve(*, from_client: DomainClient, to_client: DomainClient): + diff_state_before = compare_clients(from_client, to_client) + for obj_diff_batch in diff_state_before.batches: + widget = resolve_single(obj_diff_batch) + widget.click_share_all_private_data() + res = widget.click_sync() + assert isinstance(res, SyftSuccess) + from_client.refresh() + to_client.refresh() + diff_state_after = compare_clients(from_client, to_client) + return diff_state_before, diff_state_after + + +def run_and_accept_result(client): + job_high = client.code.compute(blocking=True) + client.requests[0].accept_by_depositing_result(job_high) + return job_high + + +@syft.syft_function_single_use() +def compute() -> int: + return 42 + + +def get_ds_client(client: DomainClient) -> DomainClient: + client.register( + name="a", + email="a@a.com", + password="asdf", + password_verify="asdf", + ) + return client.login(email="a@a.com", password="asdf") @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") @@ -212,9 +249,9 @@ def compute_mean(data) -> float: def test_diff_state(low_worker, high_worker): - low_client = low_worker.root_client - client_low_ds = low_worker.guest_client - high_client = high_worker.root_client + low_client: DomainClient = low_worker.root_client + client_low_ds = get_ds_client(low_client) + high_client: DomainClient = high_worker.root_client @sy.syft_function_single_use() def compute() -> int: @@ -224,26 +261,28 @@ def compute() -> int: _ = client_low_ds.code.request_code_execution(compute) - diff_state = compare_clients(low_client, high_client) - low_items_to_sync, high_items_to_sync = resolve( - diff_state, decision="low", share_private_objects=True + diff_state_before, diff_state_after = compare_and_resolve( + from_client=low_client, to_client=high_client ) - assert not diff_state.is_same - assert not low_items_to_sync.is_empty - assert not high_items_to_sync.is_empty + assert not diff_state_before.is_same - low_client.apply_state(low_items_to_sync) - high_client.apply_state(high_items_to_sync) + assert diff_state_after.is_same - diff_state = compare_clients(low_client, high_client) - low_items_to_sync, high_items_to_sync = resolve( - diff_state, decision="low", share_private_objects=True + run_and_accept_result(high_client) + diff_state_before, diff_state_after = compare_and_resolve( + from_client=high_client, to_client=low_client ) - assert diff_state.is_same - assert low_items_to_sync.is_empty - assert high_items_to_sync.is_empty + high_state = high_client.get_sync_state() + low_state = high_client.get_sync_state() + assert high_state.get_previous_state_diff().is_same + assert low_state.get_previous_state_diff().is_same + assert diff_state_after.is_same + + client_low_ds.refresh() + res = client_low_ds.code.compute(blocking=True) + assert res == compute(blocking=True).get() def test_forget_usercode(low_worker, high_worker): @@ -306,18 +345,7 @@ def private_function(context) -> str: def test_twin_api_integration(low_worker, high_worker): low_client = low_worker.root_client high_client = high_worker.root_client - - low_client.register( - email="newuser@openmined.org", - name="John Doe", - password="pw", - password_verify="pw", - ) - - client_low_ds = low_client.login( - email="newuser@openmined.org", - password="pw", - ) + client_low_ds = get_ds_client(low_client) new_endpoint = sy.TwinAPIEndpoint( path="testapi.query", @@ -330,12 +358,9 @@ def test_twin_api_integration(low_worker, high_worker): high_private_res = high_client.api.services.testapi.query.private() assert high_private_res == 42 - low_state = low_client.get_sync_state() - high_state = high_client.get_sync_state() - diff_state = compare_states(high_state, low_state) - obj_diff_batch = diff_state[0] - widget = resolve_single(obj_diff_batch) - widget.click_sync() + diff_before, diff_after = compare_and_resolve( + from_client=high_client, to_client=low_client + ) client_low_ds.refresh() low_private_res = client_low_ds.api.services.testapi.query.private() @@ -346,6 +371,29 @@ def test_twin_api_integration(low_worker, high_worker): high_mock_res = high_client.api.services.testapi.query.mock() assert low_mock_res == high_mock_res == -42 + @syft.syft_function_single_use( + query=client_low_ds.api.services.testapi.query, + ) + def compute(query): + return query() + + compute.code = dedent(compute.code) + _ = client_low_ds.code.request_code_execution(compute) + + diff_before, diff_after = compare_and_resolve( + from_client=low_client, to_client=high_client + ) + + job_high = high_client.code.compute(query=high_client.api.services.testapi.query) + high_client.requests[0].accept_by_depositing_result(job_high) + diff_before, diff_after = compare_and_resolve( + from_client=high_client, to_client=low_client + ) + client_low_ds.refresh() + res = client_low_ds.code.compute(query=client_low_ds.api.services.testapi.query) + assert res.syft_action_data == high_client.api.services.testapi.query.private() + assert diff_after.is_same + def test_skip_user_code(low_worker, high_worker): low_client = low_worker.root_client diff --git a/packages/syft/tests/syft/service/sync/sync_widget_test.py b/packages/syft/tests/syft/service/sync/sync_widget_test.py index 81e0db4d41b..13a4f5cb110 100644 --- a/packages/syft/tests/syft/service/sync/sync_widget_test.py +++ b/packages/syft/tests/syft/service/sync/sync_widget_test.py @@ -1,9 +1,13 @@ +# stdlib +from textwrap import dedent + # syft absolute import syft from syft.client.domain_client import DomainClient from syft.client.syncing import compare_clients from syft.client.syncing import resolve_single -from syft.service.job.job_stash import Job +from syft.service.response import SyftError +from syft.service.response import SyftSuccess def compare_and_resolve(*, from_client: DomainClient, to_client: DomainClient): @@ -11,7 +15,8 @@ def compare_and_resolve(*, from_client: DomainClient, to_client: DomainClient): for obj_diff_batch in diff_state_before.batches: widget = resolve_single(obj_diff_batch) widget.click_share_all_private_data() - widget.click_sync() + res = widget.click_sync() + assert isinstance(res, SyftSuccess) from_client.refresh() to_client.refresh() diff_state_after = compare_clients(from_client, to_client) @@ -29,9 +34,19 @@ def compute() -> int: return 42 +def get_ds_client(client: DomainClient) -> DomainClient: + client.register( + name="a", + email="a@a.com", + password="asdf", + password_verify="asdf", + ) + return client.login(email="a@a.com", password="asdf") + + def test_diff_widget_merge_status(low_worker, high_worker): low_client = low_worker.root_client - client_low_ds = low_worker.guest_client + client_low_ds = get_ds_client(low_client) high_client = high_worker.root_client _ = client_low_ds.code.request_code_execution(compute) @@ -44,6 +59,71 @@ def test_diff_widget_merge_status(low_worker, high_worker): from_client=high_client, to_client=low_client ) - # expect to have a job diff, might fail if the implementation changes - job_diff = [batch for batch in diff_after.batches if batch.root_type is Job][0] - assert job_diff.status == "SAME", "Job diff status should be SAME after syncing" + assert diff_after.is_same + + client_low_ds.refresh() + res = client_low_ds.code.compute(blocking=True) + assert res == 42 + + +@syft.mock_api_endpoint() +def mock_function(context) -> str: + return -42 + + +@syft.private_api_endpoint() +def private_function(context) -> str: + return 42 + + +def test_twin_api_integration2(low_worker, high_worker): + low_client = low_worker.root_client + high_client = high_worker.root_client + client_low_ds = get_ds_client(low_client) + + new_endpoint = syft.TwinAPIEndpoint( + path="testapi.query", + private_function=private_function, + mock_function=mock_function, + description="", + ) + high_client.api.services.api.add(endpoint=new_endpoint) + high_client.refresh() + high_private_res = high_client.api.services.testapi.query.private() + assert high_private_res == 42 + + diff_before, diff_after = compare_and_resolve( + from_client=high_client, to_client=low_client + ) + + client_low_ds.refresh() + low_private_res = client_low_ds.api.services.testapi.query.private() + assert isinstance( + low_private_res, SyftError + ), "Should not have access to private on low side" + low_mock_res = client_low_ds.api.services.testapi.query.mock() + high_mock_res = high_client.api.services.testapi.query.mock() + assert low_mock_res == high_mock_res == -42 + + @syft.syft_function_single_use( + query=client_low_ds.api.services.testapi.query, + ) + def compute(query): + return query() + + compute.code = dedent(compute.code) + _ = client_low_ds.code.request_code_execution(compute) + + diff_before, diff_after = compare_and_resolve( + from_client=low_client, to_client=high_client + ) + + job_high = high_client.code.compute(query=high_client.api.services.testapi.query) + high_client.requests[0].accept_by_depositing_result(job_high) + diff_before, diff_after = compare_and_resolve( + from_client=high_client, to_client=low_client + ) + client_low_ds.refresh() + res = client_low_ds.code.compute(query=client_low_ds.api.services.testapi.query) + assert res.syft_action_data == high_client.api.services.testapi.query.private() + assert diff_after.is_same From 824be284ab2fadb627fad87cc78d49b88fc9200b Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Thu, 18 Apr 2024 12:18:26 +0200 Subject: [PATCH 18/21] rename new test file --- .../tests/syft/service/sync/sync_flow_test.py | 100 ------------------ ...et_test.py => sync_resolve_single_test.py} | 38 +++++-- 2 files changed, 27 insertions(+), 111 deletions(-) rename packages/syft/tests/syft/service/sync/{sync_widget_test.py => sync_resolve_single_test.py} (79%) diff --git a/packages/syft/tests/syft/service/sync/sync_flow_test.py b/packages/syft/tests/syft/service/sync/sync_flow_test.py index fae5fdfb18f..432c96e3700 100644 --- a/packages/syft/tests/syft/service/sync/sync_flow_test.py +++ b/packages/syft/tests/syft/service/sync/sync_flow_test.py @@ -248,43 +248,6 @@ def compute_mean(data) -> float: high_worker.cleanup() -def test_diff_state(low_worker, high_worker): - low_client: DomainClient = low_worker.root_client - client_low_ds = get_ds_client(low_client) - high_client: DomainClient = high_worker.root_client - - @sy.syft_function_single_use() - def compute() -> int: - return 42 - - compute.code = dedent(compute.code) - - _ = client_low_ds.code.request_code_execution(compute) - - diff_state_before, diff_state_after = compare_and_resolve( - from_client=low_client, to_client=high_client - ) - - assert not diff_state_before.is_same - - assert diff_state_after.is_same - - run_and_accept_result(high_client) - diff_state_before, diff_state_after = compare_and_resolve( - from_client=high_client, to_client=low_client - ) - - high_state = high_client.get_sync_state() - low_state = high_client.get_sync_state() - assert high_state.get_previous_state_diff().is_same - assert low_state.get_previous_state_diff().is_same - assert diff_state_after.is_same - - client_low_ds.refresh() - res = client_low_ds.code.compute(blocking=True) - assert res == compute(blocking=True).get() - - def test_forget_usercode(low_worker, high_worker): low_client = low_worker.root_client client_low_ds = low_worker.guest_client @@ -332,69 +295,6 @@ def skip_if_user_code(diff): ) -@sy.mock_api_endpoint() -def mock_function(context) -> str: - return -42 - - -@sy.private_api_endpoint() -def private_function(context) -> str: - return 42 - - -def test_twin_api_integration(low_worker, high_worker): - low_client = low_worker.root_client - high_client = high_worker.root_client - client_low_ds = get_ds_client(low_client) - - new_endpoint = sy.TwinAPIEndpoint( - path="testapi.query", - private_function=private_function, - mock_function=mock_function, - description="", - ) - high_client.api.services.api.add(endpoint=new_endpoint) - high_client.refresh() - high_private_res = high_client.api.services.testapi.query.private() - assert high_private_res == 42 - - diff_before, diff_after = compare_and_resolve( - from_client=high_client, to_client=low_client - ) - - client_low_ds.refresh() - low_private_res = client_low_ds.api.services.testapi.query.private() - assert isinstance( - low_private_res, SyftError - ), "Should not have access to private on low side" - low_mock_res = client_low_ds.api.services.testapi.query.mock() - high_mock_res = high_client.api.services.testapi.query.mock() - assert low_mock_res == high_mock_res == -42 - - @syft.syft_function_single_use( - query=client_low_ds.api.services.testapi.query, - ) - def compute(query): - return query() - - compute.code = dedent(compute.code) - _ = client_low_ds.code.request_code_execution(compute) - - diff_before, diff_after = compare_and_resolve( - from_client=low_client, to_client=high_client - ) - - job_high = high_client.code.compute(query=high_client.api.services.testapi.query) - high_client.requests[0].accept_by_depositing_result(job_high) - diff_before, diff_after = compare_and_resolve( - from_client=high_client, to_client=low_client - ) - client_low_ds.refresh() - res = client_low_ds.code.compute(query=client_low_ds.api.services.testapi.query) - assert res.syft_action_data == high_client.api.services.testapi.query.private() - assert diff_after.is_same - - def test_skip_user_code(low_worker, high_worker): low_client = low_worker.root_client client_low_ds = low_worker.guest_client diff --git a/packages/syft/tests/syft/service/sync/sync_widget_test.py b/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py similarity index 79% rename from packages/syft/tests/syft/service/sync/sync_widget_test.py rename to packages/syft/tests/syft/service/sync/sync_resolve_single_test.py index 13a4f5cb110..4be107ec8ac 100644 --- a/packages/syft/tests/syft/service/sync/sync_widget_test.py +++ b/packages/syft/tests/syft/service/sync/sync_resolve_single_test.py @@ -3,6 +3,7 @@ # syft absolute import syft +import syft as sy from syft.client.domain_client import DomainClient from syft.client.syncing import compare_clients from syft.client.syncing import resolve_single @@ -44,44 +45,59 @@ def get_ds_client(client: DomainClient) -> DomainClient: return client.login(email="a@a.com", password="asdf") -def test_diff_widget_merge_status(low_worker, high_worker): - low_client = low_worker.root_client +def test_diff_state(low_worker, high_worker): + low_client: DomainClient = low_worker.root_client client_low_ds = get_ds_client(low_client) - high_client = high_worker.root_client + high_client: DomainClient = high_worker.root_client + + @sy.syft_function_single_use() + def compute() -> int: + return 42 + + compute.code = dedent(compute.code) _ = client_low_ds.code.request_code_execution(compute) - diff_before, diff_after = compare_and_resolve( + diff_state_before, diff_state_after = compare_and_resolve( from_client=low_client, to_client=high_client ) + + assert not diff_state_before.is_same + + assert diff_state_after.is_same + run_and_accept_result(high_client) - diff_before, diff_after = compare_and_resolve( + diff_state_before, diff_state_after = compare_and_resolve( from_client=high_client, to_client=low_client ) - assert diff_after.is_same + high_state = high_client.get_sync_state() + low_state = high_client.get_sync_state() + assert high_state.get_previous_state_diff().is_same + assert low_state.get_previous_state_diff().is_same + assert diff_state_after.is_same client_low_ds.refresh() res = client_low_ds.code.compute(blocking=True) - assert res == 42 + assert res == compute(blocking=True).get() -@syft.mock_api_endpoint() +@sy.mock_api_endpoint() def mock_function(context) -> str: return -42 -@syft.private_api_endpoint() +@sy.private_api_endpoint() def private_function(context) -> str: return 42 -def test_twin_api_integration2(low_worker, high_worker): +def test_twin_api_integration(low_worker, high_worker): low_client = low_worker.root_client high_client = high_worker.root_client client_low_ds = get_ds_client(low_client) - new_endpoint = syft.TwinAPIEndpoint( + new_endpoint = sy.TwinAPIEndpoint( path="testapi.query", private_function=private_function, mock_function=mock_function, From 708964f02d25d2f4b1c09e975c8c28d40ba31bd0 Mon Sep 17 00:00:00 2001 From: Aziz Berkay Yesilyurt Date: Thu, 18 Apr 2024 12:25:45 +0200 Subject: [PATCH 19/21] remove change --- packages/syft/src/syft/service/action/action_object.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/syft/src/syft/service/action/action_object.py b/packages/syft/src/syft/service/action/action_object.py index f03afdd2686..f77d7a8330e 100644 --- a/packages/syft/src/syft/service/action/action_object.py +++ b/packages/syft/src/syft/service/action/action_object.py @@ -2078,11 +2078,6 @@ def __float__(self) -> float: def __int__(self) -> float: return int(self.syft_action_data) - def syft_eq(self, ext_obj: Self | None) -> bool: - if ext_obj is None: - return False - return self.id.id == ext_obj.id.id - action_types[Any] = AnyActionObject From dc7b2642707938312bda1c2a822fdb494fa221c4 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Thu, 18 Apr 2024 13:14:52 +0200 Subject: [PATCH 20/21] fix part size --- packages/grid/devspace.yaml | 2 +- packages/grid/helm/values.dev.high.yaml | 44 +++++++++++++++++++ packages/grid/helm/values.dev.low.yaml | 44 +++++++++++++++++++ packages/syft/setup.cfg | 1 + .../src/syft/store/blob_storage/seaweedfs.py | 2 +- 5 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 packages/grid/helm/values.dev.high.yaml create mode 100644 packages/grid/helm/values.dev.low.yaml diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 473a9d82e24..1b302389d02 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -72,7 +72,7 @@ deployments: type: domain # required for the gateway profile # anything that does not need devspace $env vars should go in values.dev.yaml valuesFiles: - - ./helm/values.dev.yaml + - ./helm/values.dev.low.yaml dev: mongo: diff --git a/packages/grid/helm/values.dev.high.yaml b/packages/grid/helm/values.dev.high.yaml new file mode 100644 index 00000000000..62b9fddb9fd --- /dev/null +++ b/packages/grid/helm/values.dev.high.yaml @@ -0,0 +1,44 @@ +# Helm chart values used for development and testing +# Can be used through `helm install -f values.dev.yaml` or devspace `valuesFiles` + +global: + randomizedSecrets: false + +registry: + resourcesPreset: null + resources: null + + storageSize: "5Gi" + +node: + rootEmail: info@openmined.org + defaultWorkerPoolCount: 1 + side: high + + resourcesPreset: 2xlarge + resources: null + + secret: + defaultRootPassword: changethis + +mongo: + resourcesPreset: null + resources: null + + secret: + rootPassword: example + +seaweedfs: + resourcesPreset: null + resources: null + + secret: + s3RootPassword: admin + +frontend: + resourcesPreset: null + resources: null + +proxy: + resourcesPreset: null + resources: null diff --git a/packages/grid/helm/values.dev.low.yaml b/packages/grid/helm/values.dev.low.yaml new file mode 100644 index 00000000000..62d63c4f73b --- /dev/null +++ b/packages/grid/helm/values.dev.low.yaml @@ -0,0 +1,44 @@ +# Helm chart values used for development and testing +# Can be used through `helm install -f values.dev.yaml` or devspace `valuesFiles` + +global: + randomizedSecrets: false + +registry: + resourcesPreset: null + resources: null + + storageSize: "5Gi" + +node: + rootEmail: info@openmined.org + defaultWorkerPoolCount: 1 + side: low + + resourcesPreset: 2xlarge + resources: null + + secret: + defaultRootPassword: changethis + +mongo: + resourcesPreset: null + resources: null + + secret: + rootPassword: example + +seaweedfs: + resourcesPreset: null + resources: null + + secret: + s3RootPassword: admin + +frontend: + resourcesPreset: null + resources: null + +proxy: + resourcesPreset: null + resources: null diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 6147adce6d0..6c5b048631b 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -68,6 +68,7 @@ syft = PyYAML==6.0.1 azure-storage-blob==12.19.1 ipywidgets==8.1.2 + google-cloud-bigquery[pandas] install_requires = %(syft)s diff --git a/packages/syft/src/syft/store/blob_storage/seaweedfs.py b/packages/syft/src/syft/store/blob_storage/seaweedfs.py index c3a7d4c7ff7..74762c4155f 100644 --- a/packages/syft/src/syft/store/blob_storage/seaweedfs.py +++ b/packages/syft/src/syft/store/blob_storage/seaweedfs.py @@ -37,7 +37,7 @@ MAX_QUEUE_SIZE = 100 WRITE_EXPIRATION_TIME = 900 # seconds -DEFAULT_FILE_PART_SIZE = 1024**3 / 100 # 10MB +DEFAULT_FILE_PART_SIZE = 1024**3 # 1GB DEFAULT_UPLOAD_CHUNK_SIZE = 1024 * 800 # 800KB From 02e53959a04ab28c94ffb88fa6efdab4617f9e54 Mon Sep 17 00:00:00 2001 From: eelcovdw Date: Thu, 18 Apr 2024 13:18:34 +0200 Subject: [PATCH 21/21] revert --- packages/grid/devspace.yaml | 2 +- packages/syft/setup.cfg | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 1b302389d02..473a9d82e24 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -72,7 +72,7 @@ deployments: type: domain # required for the gateway profile # anything that does not need devspace $env vars should go in values.dev.yaml valuesFiles: - - ./helm/values.dev.low.yaml + - ./helm/values.dev.yaml dev: mongo: diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 6c5b048631b..6147adce6d0 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -68,7 +68,6 @@ syft = PyYAML==6.0.1 azure-storage-blob==12.19.1 ipywidgets==8.1.2 - google-cloud-bigquery[pandas] install_requires = %(syft)s