From 85cb06ebfe4b9f1a5a0c4c84c5350bbfb81dd726 Mon Sep 17 00:00:00 2001 From: Dan King Date: Wed, 26 Apr 2023 15:49:44 -0400 Subject: [PATCH 01/26] [build.yaml][ci] upgrade to skopeo 1.12.0 (#12937) --- build.yaml | 4 ++-- ci/Dockerfile.ci-utils | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.yaml b/build.yaml index 25e803181d4..a601cb2ef20 100644 --- a/build.yaml +++ b/build.yaml @@ -61,7 +61,7 @@ steps: - azure - kind: runImage name: copy_third_party_images - image: quay.io/skopeo/stable:v1.11.1 + image: quay.io/skopeo/stable:v1.12.0 script: | set -ex @@ -3270,7 +3270,7 @@ steps: - gcp - kind: runImage name: mirror_hailgenetics_images - image: quay.io/skopeo/stable:v1.11.1 + image: quay.io/skopeo/stable:v1.12.0 script: | set -ex diff --git a/ci/Dockerfile.ci-utils b/ci/Dockerfile.ci-utils index 2f4d0f4ca2b..b0011a793b4 100644 --- a/ci/Dockerfile.ci-utils +++ b/ci/Dockerfile.ci-utils @@ -26,7 +26,7 @@ FROM golang:1.18 AS skopeo-build WORKDIR /usr/src/skopeo -ARG SKOPEO_VERSION="1.8.0" +ARG SKOPEO_VERSION="1.12.0" RUN curl -fsSL "https://github.com/containers/skopeo/archive/v${SKOPEO_VERSION}.tar.gz" \ | tar -xzf - --strip-components=1 From 97015b17a97e29725cd3307db0fa75f53f716178 Mon Sep 17 00:00:00 2001 From: Dan King Date: Wed, 26 Apr 2023 16:51:54 -0400 Subject: [PATCH 02/26] [terraform] align gcp terraform with current state & create new terraform project to track true current state (#12882) I fixed some issues in `gcp` but there are enough difficult to resolve issues that I want to start a fresh terraform project to track reality as different from what we recommend a new user do. The fresh attempt project is `gcp-broad`. I plan to slowly expand that until it captures everything. From this point forward, for the subset of infrastructure in `gcp-broad`, we should not make manual changes. Instead, we should propose changes as PRs, review them, merge them, and apply the terraform by hand once the changes are in `main`. --- build.yaml | 4 - infra/gcp-broad/README.md | 5 + infra/gcp-broad/ci/main.tf | 81 +++ infra/gcp-broad/ci/variables.tf | 37 ++ infra/gcp-broad/gcs_bucket/main.tf | 10 + infra/gcp-broad/gcs_bucket/outputs.tf | 3 + infra/gcp-broad/gcs_bucket/variables.tf | 11 + infra/gcp-broad/gsa/main.tf | 27 + infra/gcp-broad/gsa/outputs.tf | 3 + infra/gcp-broad/gsa/variables.tf | 12 + infra/gcp-broad/main.tf | 667 ++++++++++++++++++++++++ infra/gcp-broad/outputs.tf | 3 + 12 files changed, 859 insertions(+), 4 deletions(-) create mode 100644 infra/gcp-broad/README.md create mode 100644 infra/gcp-broad/ci/main.tf create mode 100644 infra/gcp-broad/ci/variables.tf create mode 100644 infra/gcp-broad/gcs_bucket/main.tf create mode 100644 infra/gcp-broad/gcs_bucket/outputs.tf create mode 100644 infra/gcp-broad/gcs_bucket/variables.tf create mode 100644 infra/gcp-broad/gsa/main.tf create mode 100644 infra/gcp-broad/gsa/outputs.tf create mode 100644 infra/gcp-broad/gsa/variables.tf create mode 100644 infra/gcp-broad/main.tf create mode 100644 infra/gcp-broad/outputs.tf diff --git a/build.yaml b/build.yaml index a601cb2ef20..dfcf1188a4d 100644 --- a/build.yaml +++ b/build.yaml @@ -2756,10 +2756,6 @@ steps: namespace: valueFrom: default_ns.name mountPath: /user-tokens - - name: hail-ci-0-1-service-account-key - namespace: - valueFrom: default_ns.name - mountPath: /secret/ci-secrets - name: ssl-config-ci-tests namespace: valueFrom: default_ns.name diff --git a/infra/gcp-broad/README.md b/infra/gcp-broad/README.md new file mode 100644 index 00000000000..b4537220d69 --- /dev/null +++ b/infra/gcp-broad/README.md @@ -0,0 +1,5 @@ +If you're a third-party trying to deploy Hail, look at `../gcp`. + +Hail team, this directory is an underestimate of our infrastructure. We are iteratively adding more +infrastructure. Infrastructure may be missing because importing into terraform would require a +destroy/create. diff --git a/infra/gcp-broad/ci/main.tf b/infra/gcp-broad/ci/main.tf new file mode 100644 index 00000000000..f488c1b4301 --- /dev/null +++ b/infra/gcp-broad/ci/main.tf @@ -0,0 +1,81 @@ +resource "random_string" "hail_ci_bucket_suffix" { + length = 5 +} + +resource "google_storage_bucket" "bucket" { + name = "hail-ci-${random_string.hail_ci_bucket_suffix.result}" + location = var.bucket_location + force_destroy = false + storage_class = var.bucket_storage_class + labels = { + "name" = "hail-ci-${random_string.hail_ci_bucket_suffix.result}" + } + lifecycle_rule { + action { + type = "Delete" + } + condition { + age = 7 + days_since_custom_time = 0 + days_since_noncurrent_time = 0 + matches_prefix = [] + matches_storage_class = [] + matches_suffix = [] + num_newer_versions = 0 + with_state = "ANY" + } + } + + timeouts {} +} + +resource "google_storage_bucket_iam_member" "ci_bucket_admin" { + bucket = google_storage_bucket.bucket.name + role = "roles/storage.legacyBucketWriter" + member = "serviceAccount:${var.ci_email}" +} + +resource "kubernetes_secret" "ci_config" { + metadata { + name = "ci-config" + } + + data = { + storage_uri = "gs://${google_storage_bucket.bucket.name}" + deploy_steps = jsonencode(var.deploy_steps) + watched_branches = jsonencode(var.watched_branches) + github_context = var.github_context + test_oauth2_callback_urls = var.test_oauth2_callback_urls + } +} + +resource "kubernetes_secret" "zulip_config" { + count = fileexists("~/.hail/.zuliprc") ? 1 : 0 + metadata { + name = "zulip-config" + } + + data = { + ".zuliprc" = fileexists("~/.hail/.zuliprc") ? file("~/.hail/.zuliprc") : "" + } +} + +resource "kubernetes_secret" "hail_ci_0_1_github_oauth_token" { + metadata { + name = "hail-ci-0-1-github-oauth-token" + } + + data = { + "oauth-token" = var.github_oauth_token + } +} + +resource "kubernetes_secret" "hail_ci_0_1_service_account_key" { + metadata { + name = "hail-ci-0-1-service-account-key" + } + + data = { + "user1" = var.github_user1_oauth_token + } +} diff --git a/infra/gcp-broad/ci/variables.tf b/infra/gcp-broad/ci/variables.tf new file mode 100644 index 00000000000..f9301e1b81f --- /dev/null +++ b/infra/gcp-broad/ci/variables.tf @@ -0,0 +1,37 @@ +variable "github_oauth_token" { + type = string + sensitive = true +} + +variable "github_user1_oauth_token" { + type = string + sensitive = true +} + +variable "bucket_location" { + type = string +} + +variable "bucket_storage_class" { + type = string +} + +variable "watched_branches" { + type = list(tuple([string, bool, bool])) +} + +variable "deploy_steps" { + type = list(string) +} + +variable "ci_email" { + type = string +} + +variable "github_context" { + type = string +} + +variable "test_oauth2_callback_urls" { + type = string +} diff --git a/infra/gcp-broad/gcs_bucket/main.tf b/infra/gcp-broad/gcs_bucket/main.tf new file mode 100644 index 00000000000..e56ee3430b6 --- /dev/null +++ b/infra/gcp-broad/gcs_bucket/main.tf @@ -0,0 +1,10 @@ +resource "random_string" "bucket_suffix" { + length = 5 +} + +resource "google_storage_bucket" "bucket" { + name = "${var.short_name}-${random_string.bucket_suffix.result}" + location = var.location + force_destroy = true + storage_class = var.storage_class +} diff --git a/infra/gcp-broad/gcs_bucket/outputs.tf b/infra/gcp-broad/gcs_bucket/outputs.tf new file mode 100644 index 00000000000..8e191c03461 --- /dev/null +++ b/infra/gcp-broad/gcs_bucket/outputs.tf @@ -0,0 +1,3 @@ +output "name" { + value = google_storage_bucket.bucket.name +} diff --git a/infra/gcp-broad/gcs_bucket/variables.tf b/infra/gcp-broad/gcs_bucket/variables.tf new file mode 100644 index 00000000000..6965da7d5b2 --- /dev/null +++ b/infra/gcp-broad/gcs_bucket/variables.tf @@ -0,0 +1,11 @@ +variable "short_name" { + type = string +} + +variable "location" { + type = string +} + +variable "storage_class" { + type = string +} diff --git a/infra/gcp-broad/gsa/main.tf b/infra/gcp-broad/gsa/main.tf new file mode 100644 index 00000000000..4f596aba644 --- /dev/null +++ b/infra/gcp-broad/gsa/main.tf @@ -0,0 +1,27 @@ +resource "random_string" "name_suffix" { + length = 3 + special = false + upper = false + lower = false + lifecycle { + ignore_changes = [ + special, + upper, + lower, + ] + } +} + +resource "google_service_account" "service_account" { + display_name = "${var.name}-${random_string.name_suffix.result}" + account_id = "${var.name}-${random_string.name_suffix.result}" + timeouts {} +} + +resource "google_project_iam_member" "iam_member" { + for_each = toset(var.iam_roles) + + project = var.project + role = "roles/${each.key}" + member = "serviceAccount:${google_service_account.service_account.email}" +} diff --git a/infra/gcp-broad/gsa/outputs.tf b/infra/gcp-broad/gsa/outputs.tf new file mode 100644 index 00000000000..ac47775f2b7 --- /dev/null +++ b/infra/gcp-broad/gsa/outputs.tf @@ -0,0 +1,3 @@ +output "email" { + value = google_service_account.service_account.email +} diff --git a/infra/gcp-broad/gsa/variables.tf b/infra/gcp-broad/gsa/variables.tf new file mode 100644 index 00000000000..ef6241f20d5 --- /dev/null +++ b/infra/gcp-broad/gsa/variables.tf @@ -0,0 +1,12 @@ +variable "name" { + type = string +} + +variable project { + type = string +} + +variable "iam_roles" { + type = list(string) + default = [] +} diff --git a/infra/gcp-broad/main.tf b/infra/gcp-broad/main.tf new file mode 100644 index 00000000000..4d68d0d7d54 --- /dev/null +++ b/infra/gcp-broad/main.tf @@ -0,0 +1,667 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "4.32.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.8.0" + } + sops = { + source = "carlpett/sops" + version = "0.6.3" + } + } +} + +variable "k8s_preemptible_node_pool_name" { + type = string + default = "preemptible-pool" +} +variable "k8s_nonpreemptible_node_pool_name" { + type = string + default = "nonpreemptible-pool" +} +variable "batch_gcp_regions" {} +variable "gcp_project" {} +variable "batch_logs_bucket_location" {} +variable "batch_logs_bucket_storage_class" {} +variable "hail_query_bucket_location" {} +variable "hail_query_bucket_storage_class" {} +variable "hail_test_gcs_bucket_location" {} +variable "hail_test_gcs_bucket_storage_class" {} +variable "gcp_region" {} +variable "gcp_zone" {} +variable "gcp_location" {} +variable "domain" {} +variable "organization_domain" {} +variable "github_organization" {} +variable "use_artifact_registry" { + type = bool + description = "pull the ubuntu image from Artifact Registry. Otherwise, GCR" +} +variable artifact_registry_location {} + +variable deploy_ukbb { + type = bool + description = "Run the UKBB Genetic Correlation browser" + default = false +} +variable default_subnet_ip_cidr_range {} + +locals { + docker_prefix = ( + var.use_artifact_registry ? + "${var.gcp_region}-docker.pkg.dev/${var.gcp_project}/hail" : + "gcr.io/${var.gcp_project}" + ) + docker_root_image = "${local.docker_prefix}/ubuntu:20.04" +} + +provider "google" { + project = var.gcp_project + region = var.gcp_region + zone = var.gcp_zone +} + +provider "google-beta" { + project = var.gcp_project + region = var.gcp_region + zone = var.gcp_zone +} + +data "google_client_config" "provider" {} + +resource "google_project_service" "service_networking" { + disable_on_destroy = false + service = "servicenetworking.googleapis.com" + timeouts {} +} + +resource "google_compute_network" "default" { + name = "default" + description = "Default network for the project" + enable_ula_internal_ipv6 = false +} + +resource "google_compute_subnetwork" "default_region" { + name = "default" + region = var.gcp_region + network = google_compute_network.default.id + ip_cidr_range = var.default_subnet_ip_cidr_range + private_ip_google_access = true + + timeouts {} +} + +resource "google_container_cluster" "vdc" { + provider = google-beta + name = "vdc" + location = var.gcp_zone + network = google_compute_network.default.name + enable_shielded_nodes = false + + # We can't create a cluster with no node pool defined, but we want to only use + # separately managed node pools. So we create the smallest possible default + # node pool and immediately delete it. + # remove_default_node_pool = true + remove_default_node_pool = null + initial_node_count = 0 + + resource_labels = { + role = "vdc" + } + + release_channel { + channel = "REGULAR" + } + + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } + + cluster_autoscaling { + # Don't use node auto-provisioning since we manage node pools ourselves + enabled = false + autoscaling_profile = "OPTIMIZE_UTILIZATION" + } + + resource_usage_export_config { + enable_network_egress_metering = false + enable_resource_consumption_metering = true + + bigquery_destination { + dataset_id = "gke_vdc_usage" + } + } + + timeouts {} +} + +resource "google_container_node_pool" "vdc_preemptible_pool" { + name = var.k8s_preemptible_node_pool_name + location = var.gcp_zone + cluster = google_container_cluster.vdc.name + + # Allocate at least one node, so that autoscaling can take place. + initial_node_count = 3 + + autoscaling { + min_node_count = 0 + max_node_count = 200 + } + + node_config { + spot = true + machine_type = "n1-standard-2" + + labels = { + "preemptible" = "true" + } + + taint { + key = "preemptible" + value = "true" + effect = "NO_SCHEDULE" + } + + metadata = { + disable-legacy-endpoints = "true" + } + + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/trace.append", + ] + tags = [] + + shielded_instance_config { + enable_integrity_monitoring = true + enable_secure_boot = false + } + } + + timeouts {} + + upgrade_settings { + max_surge = 1 + max_unavailable = 0 + } +} + +resource "google_container_node_pool" "vdc_nonpreemptible_pool" { + name = var.k8s_nonpreemptible_node_pool_name + location = var.gcp_zone + cluster = google_container_cluster.vdc.name + + # Allocate at least one node, so that autoscaling can take place. + initial_node_count = 2 + + autoscaling { + min_node_count = 0 + max_node_count = 200 + } + + management { + auto_repair = true + auto_upgrade = true + } + + node_config { + preemptible = false + machine_type = "n1-standard-2" + + labels = { + preemptible = "false" + } + + metadata = { + disable-legacy-endpoints = "true" + } + + oauth_scopes = [ + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/trace.append", + ] + + tags = [] + + shielded_instance_config { + enable_integrity_monitoring = true + enable_secure_boot = false + } + } + + timeouts {} + + upgrade_settings { + max_surge = 1 + max_unavailable = 0 + } +} + +resource "random_string" "db_name_suffix" { + length = 5 + special = false + numeric = false + upper = false + lifecycle { + ignore_changes = [ + special, + numeric, + upper, + ] + } +} + +resource "google_service_networking_connection" "private_vpc_connection" { + # google_compute_network returns the name as the project but our extant networking connection uses + # the number + network = "projects/859893752941/global/networks/default" # google_compute_network.default.id + service = "services/servicenetworking.googleapis.com" + reserved_peering_ranges = [ + "jg-test-clone-resource-id-ip-range", + ] + + timeouts {} +} + + +resource "google_sql_database_instance" "db" { + name = "db-${random_string.db_name_suffix.result}" + database_version = "MYSQL_8_0_28" + region = var.gcp_region + + depends_on = [google_service_networking_connection.private_vpc_connection] + + settings { + # 4 vCPU and 15360 MB + # https://cloud.google.com/sql/docs/mysql/instance-settings + tier = "db-custom-4-15360" + + ip_configuration { + ipv4_enabled = false + private_network = google_compute_network.default.id + require_ssl = true + } + + backup_configuration { + binary_log_enabled = false + enabled = true + location = "us" + point_in_time_recovery_enabled = false + start_time = "13:00" + transaction_log_retention_days = 7 + + backup_retention_settings { + retained_backups = 7 + retention_unit = "COUNT" + } + } + + database_flags { + name = "innodb_log_buffer_size" + value = "536870912" + } + database_flags { + name = "innodb_log_file_size" + value = "5368709120" + } + database_flags { + name = "event_scheduler" + value = "on" + } + database_flags { + name = "skip_show_database" + value = "on" + } + database_flags { + name = "local_infile" + value = "off" + } + + insights_config { + query_insights_enabled = true + query_string_length = 1024 + record_application_tags = false + record_client_address = false + } + + location_preference { + zone = "us-central1-a" + } + + maintenance_window { + day = 7 + hour = 16 + } + } + + timeouts {} +} + +resource "google_compute_address" "gateway" { + name = "site" + region = var.gcp_region +} + +resource "google_compute_address" "internal_gateway" { + name = "internal-gateway" + # subnetwork = data.google_compute_subnetwork.default_region.id + address_type = "INTERNAL" + region = var.gcp_region +} + +provider "kubernetes" { + host = "https://${google_container_cluster.vdc.endpoint}" + token = data.google_client_config.provider.access_token + cluster_ca_certificate = base64decode( + google_container_cluster.vdc.master_auth[0].cluster_ca_certificate, + ) +} + +resource "google_artifact_registry_repository" "repository" { + provider = google-beta + format = "DOCKER" + repository_id = "hail" + location = var.artifact_registry_location +} + +resource "google_service_account" "gcr_push" { + account_id = "gcr-push" + display_name = "push to gcr.io" +} + +resource "google_artifact_registry_repository_iam_member" "artifact_registry_batch_agent_viewer" { + provider = google-beta + project = var.gcp_project + repository = google_artifact_registry_repository.repository.name + location = var.artifact_registry_location + role = "roles/artifactregistry.reader" + member = "serviceAccount:${google_service_account.batch_agent.email}" +} + +resource "google_artifact_registry_repository_iam_member" "artifact_registry_ci_viewer" { + provider = google-beta + project = var.gcp_project + repository = google_artifact_registry_repository.repository.name + location = var.artifact_registry_location + role = "roles/artifactregistry.reader" + member = "serviceAccount:${module.ci_gsa_secret.email}" +} + +resource "google_artifact_registry_repository_iam_member" "artifact_registry_push_admin" { + provider = google-beta + project = var.gcp_project + repository = google_artifact_registry_repository.repository.name + location = var.artifact_registry_location + role = "roles/artifactregistry.repoAdmin" + member = "serviceAccount:${google_service_account.gcr_push.email}" +} + +module "ukbb" { + count = var.deploy_ukbb ? 1 : 0 + source = "../k8s/ukbb" +} + +module "auth_gsa_secret" { + source = "./gsa" + name = "auth" + project = var.gcp_project + iam_roles = [ + "iam.serviceAccountAdmin", + "iam.serviceAccountKeyAdmin", + ] +} + +module "batch_gsa_secret" { + source = "./gsa" + name = "batch" + project = var.gcp_project + iam_roles = [ + "compute.instanceAdmin.v1", + "iam.serviceAccountUser", + "logging.viewer", + ] +} + +resource "google_storage_bucket_iam_member" "batch_hail_query_bucket_storage_viewer" { + bucket = google_storage_bucket.hail_query.name + role = "roles/storage.objectViewer" + member = "serviceAccount:${module.batch_gsa_secret.email}" +} + +module "ci_gsa_secret" { + source = "./gsa" + name = "ci" + project = var.gcp_project +} + +resource "google_artifact_registry_repository_iam_member" "artifact_registry_viewer" { + provider = google-beta + project = var.gcp_project + repository = google_artifact_registry_repository.repository.name + location = var.artifact_registry_location + role = "roles/artifactregistry.reader" + member = "serviceAccount:${module.ci_gsa_secret.email}" +} + +module "grafana_gsa_secret" { + source = "./gsa" + name = "grafana" + project = var.gcp_project +} + +module "test_gsa_secret" { + source = "./gsa" + name = "test" + project = var.gcp_project + iam_roles = [ + "compute.instanceAdmin.v1", + "iam.serviceAccountUser", + "logging.viewer", + "serviceusage.serviceUsageConsumer", + ] +} + +resource "google_storage_bucket_iam_member" "test_bucket_admin" { + bucket = google_storage_bucket.hail_test_bucket.name + role = "roles/storage.admin" + member = "serviceAccount:${module.test_gsa_secret.email}" +} + +resource "google_service_account" "batch_agent" { + description = "Delete instances and pull images" + display_name = "batch2-agent" + account_id = "batch2-agent" +} + +resource "google_project_iam_member" "batch_agent_iam_member" { + for_each = toset([ + "compute.instanceAdmin.v1", + "iam.serviceAccountUser", + "logging.logWriter", + "storage.objectCreator", + "storage.objectViewer", + ]) + + project = var.gcp_project + role = "roles/${each.key}" + member = "serviceAccount:${google_service_account.batch_agent.email}" +} + +resource "google_compute_firewall" "default_allow_internal" { + name = "default-allow-internal" + network = google_compute_network.default.name + + priority = 1000 + + source_ranges = ["10.128.0.0/9"] + + allow { + ports = [] + protocol = "all" + } + + timeouts {} +} + +resource "google_compute_firewall" "vdc_to_batch_worker" { + name = "vdc-to-batch-worker" + network = google_compute_network.default.name + + source_ranges = [google_container_cluster.vdc.cluster_ipv4_cidr] + + target_tags = ["batch2-agent"] + + allow { + protocol = "icmp" + } + + allow { + protocol = "tcp" + ports = ["1-65535"] + } + + allow { + protocol = "udp" + ports = ["1-65535"] + } +} + +resource "google_storage_bucket" "batch_logs" { + name = "hail-batch" + location = var.batch_logs_bucket_location + storage_class = var.batch_logs_bucket_storage_class + labels = { + "name" = "hail-batch" + } + timeouts {} +} + + +resource "google_storage_bucket" "hail_query" { + name = "hail-query" + location = var.hail_query_bucket_location + storage_class = var.hail_query_bucket_storage_class + labels = { + "name" = "hail-query" + } + timeouts {} +} + +resource "random_string" "hail_test_bucket_suffix" { + length = 5 +} + +resource "google_storage_bucket" "hail_test_bucket" { + name = "hail-test-${random_string.hail_test_bucket_suffix.result}" + location = var.hail_test_gcs_bucket_location + force_destroy = false + storage_class = var.hail_test_gcs_bucket_storage_class + lifecycle_rule { + action { + type = "Delete" + } + condition { + age = 1 + days_since_custom_time = 0 + days_since_noncurrent_time = 0 + matches_prefix = [] + matches_storage_class = [] + matches_suffix = [] + num_newer_versions = 0 + with_state = "ANY" + } + } + + timeouts {} +} + +resource "google_dns_managed_zone" "dns_zone" { + description = "" + name = "hail" + dns_name = "hail." + visibility = "private" + + private_visibility_config { + networks { + network_url = google_compute_network.default.self_link + } + } + + timeouts {} +} + +resource "google_dns_record_set" "internal_gateway" { + name = "*.${google_dns_managed_zone.dns_zone.dns_name}" + managed_zone = google_dns_managed_zone.dns_zone.name + type = "A" + ttl = 3600 + + rrdatas = [google_compute_address.internal_gateway.address] +} + +resource "kubernetes_cluster_role" "batch" { + metadata { + name = "batch" + } + + rule { + api_groups = [""] + resources = ["secrets", "serviceaccounts"] + verbs = ["get", "list"] + } +} + +resource "kubernetes_cluster_role_binding" "batch" { + metadata { + name = "batch" + } + role_ref { + kind = "ClusterRole" + name = "batch" + api_group = "rbac.authorization.k8s.io" + } + subject { + kind = "ServiceAccount" + name = "batch" + namespace = "default" + } +} + +data "sops_file" "ci_config_sops" { + count = fileexists("${var.github_organization}/ci_config.enc.json") ? 1 : 0 + source_file = "${var.github_organization}/ci_config.enc.json" +} + +locals { + ci_config = length(data.sops_file.ci_config_sops) == 1 ? data.sops_file.ci_config_sops[0] : null +} + +module "ci" { + source = "./ci" + count = local.ci_config != null ? 1 : 0 + + github_oauth_token = local.ci_config.data["github_oauth_token"] + github_user1_oauth_token = local.ci_config.data["github_user1_oauth_token"] + watched_branches = jsondecode(local.ci_config.raw).watched_branches + deploy_steps = jsondecode(local.ci_config.raw).deploy_steps + bucket_location = local.ci_config.data["bucket_location"] + bucket_storage_class = local.ci_config.data["bucket_storage_class"] + + ci_email = module.ci_gsa_secret.email + github_context = local.ci_config.data["github_context"] + test_oauth2_callback_urls = local.ci_config.data["test_oauth2_callback_urls"] +} diff --git a/infra/gcp-broad/outputs.tf b/infra/gcp-broad/outputs.tf new file mode 100644 index 00000000000..05d76e772d0 --- /dev/null +++ b/infra/gcp-broad/outputs.tf @@ -0,0 +1,3 @@ +output "k8s_server_ip" { + value = google_container_cluster.vdc.endpoint +} From 91a383049ac6b30a5f233dceab9831c816d2c3fc Mon Sep 17 00:00:00 2001 From: Dan King Date: Wed, 26 Apr 2023 17:55:34 -0400 Subject: [PATCH 03/26] [query] revert to skopeo 1.11.2 (#12938) I should have checked quay.io for 1.12.0, which doesn't exist: https://quay.io/repository/skopeo/stable?tab=tags&tag=v1.12.0 Even though 1.12.0 was released two weeks ago https://github.com/containers/skopeo/releases --- build.yaml | 4 ++-- ci/Dockerfile.ci-utils | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build.yaml b/build.yaml index dfcf1188a4d..299c3891d96 100644 --- a/build.yaml +++ b/build.yaml @@ -61,7 +61,7 @@ steps: - azure - kind: runImage name: copy_third_party_images - image: quay.io/skopeo/stable:v1.12.0 + image: quay.io/skopeo/stable:v1.11.2 script: | set -ex @@ -3266,7 +3266,7 @@ steps: - gcp - kind: runImage name: mirror_hailgenetics_images - image: quay.io/skopeo/stable:v1.12.0 + image: quay.io/skopeo/stable:v1.11.2 script: | set -ex diff --git a/ci/Dockerfile.ci-utils b/ci/Dockerfile.ci-utils index b0011a793b4..aefd0f0b7f8 100644 --- a/ci/Dockerfile.ci-utils +++ b/ci/Dockerfile.ci-utils @@ -26,7 +26,7 @@ FROM golang:1.18 AS skopeo-build WORKDIR /usr/src/skopeo -ARG SKOPEO_VERSION="1.12.0" +ARG SKOPEO_VERSION="1.11.2" RUN curl -fsSL "https://github.com/containers/skopeo/archive/v${SKOPEO_VERSION}.tar.gz" \ | tar -xzf - --strip-components=1 From c4d21398153ba5e2e609a008264a43c4dc3a9e50 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Wed, 26 Apr 2023 19:17:41 -0400 Subject: [PATCH 04/26] [batch] Remove unused fields on cloud user credentials (#12939) `credentials_host_file_path` is unused in `main`. Deleting that allowed me to then delete `credentials_host_dirname`, after which `CloudUserCredentials.file_name` and `CloudUserCredentials.secret_name` were only used for `CloudUserCredentials.mount_path` so I merged those. `CloudUserCredentials.secret_data` and `CloudUserCredentials.hail_env_name` were unused. --- batch/batch/cloud/azure/worker/credentials.py | 20 ++-------------- batch/batch/cloud/azure/worker/worker_api.py | 2 +- batch/batch/cloud/gcp/worker/credentials.py | 22 +++--------------- batch/batch/cloud/gcp/worker/worker_api.py | 2 +- batch/batch/worker/credentials.py | 23 ++++--------------- batch/batch/worker/worker.py | 6 ----- batch/batch/worker/worker_api.py | 2 +- 7 files changed, 13 insertions(+), 64 deletions(-) diff --git a/batch/batch/cloud/azure/worker/credentials.py b/batch/batch/cloud/azure/worker/credentials.py index d2daebd39a2..da25309b8c6 100644 --- a/batch/batch/cloud/azure/worker/credentials.py +++ b/batch/batch/cloud/azure/worker/credentials.py @@ -6,30 +6,14 @@ class AzureUserCredentials(CloudUserCredentials): - def __init__(self, data: Dict[str, bytes]): + def __init__(self, data: Dict[str, str]): self._data = data self._credentials = json.loads(base64.b64decode(data['key.json']).decode()) - @property - def secret_name(self) -> str: - return 'azure-credentials' - - @property - def secret_data(self) -> Dict[str, bytes]: - return self._data - - @property - def file_name(self) -> str: - return 'key.json' - @property def cloud_env_name(self) -> str: return 'AZURE_APPLICATION_CREDENTIALS' - @property - def hail_env_name(self) -> str: - return 'HAIL_AZURE_CREDENTIAL_FILE' - @property def username(self): return self._credentials['appId'] @@ -40,7 +24,7 @@ def password(self): @property def mount_path(self): - return f'/{self.secret_name}/{self.file_name}' + return '/azure-credentials/key.json' def cloudfuse_credentials(self, fuse_config: dict) -> str: # https://github.com/Azure/azure-storage-fuse diff --git a/batch/batch/cloud/azure/worker/worker_api.py b/batch/batch/cloud/azure/worker/worker_api.py index 7a7badd081a..84a1dc6ea53 100644 --- a/batch/batch/cloud/azure/worker/worker_api.py +++ b/batch/batch/cloud/azure/worker/worker_api.py @@ -42,7 +42,7 @@ def get_compute_client(self) -> aioazure.AzureComputeClient: azure_config = get_azure_config() return aioazure.AzureComputeClient(azure_config.subscription_id, azure_config.resource_group) - def user_credentials(self, credentials: Dict[str, bytes]) -> AzureUserCredentials: + def user_credentials(self, credentials: Dict[str, str]) -> AzureUserCredentials: return AzureUserCredentials(credentials) async def worker_access_token(self, session: httpx.ClientSession) -> Dict[str, str]: diff --git a/batch/batch/cloud/gcp/worker/credentials.py b/batch/batch/cloud/gcp/worker/credentials.py index 8bb83541f1e..eef29b6b84c 100644 --- a/batch/batch/cloud/gcp/worker/credentials.py +++ b/batch/batch/cloud/gcp/worker/credentials.py @@ -5,40 +5,24 @@ class GCPUserCredentials(CloudUserCredentials): - def __init__(self, data: Dict[str, bytes]): + def __init__(self, data: Dict[str, str]): self._data = data - @property - def secret_name(self) -> str: - return 'gsa-key' - - @property - def secret_data(self) -> Dict[str, bytes]: - return self._data - - @property - def file_name(self) -> str: - return 'key.json' - @property def cloud_env_name(self) -> str: return 'GOOGLE_APPLICATION_CREDENTIALS' - @property - def hail_env_name(self) -> str: - return 'HAIL_GSA_KEY_FILE' - @property def username(self): return '_json_key' @property def password(self) -> str: - return base64.b64decode(self.secret_data['key.json']).decode() + return base64.b64decode(self._data['key.json']).decode() @property def mount_path(self): - return f'/{self.secret_name}/{self.file_name}' + return '/gsa-key/key.json' def cloudfuse_credentials(self, fuse_config: dict) -> str: # pylint: disable=unused-argument return self.password diff --git a/batch/batch/cloud/gcp/worker/worker_api.py b/batch/batch/cloud/gcp/worker/worker_api.py index adb64022c61..9a31f4da740 100644 --- a/batch/batch/cloud/gcp/worker/worker_api.py +++ b/batch/batch/cloud/gcp/worker/worker_api.py @@ -47,7 +47,7 @@ def get_cloud_async_fs(self) -> aiogoogle.GoogleStorageAsyncFS: def get_compute_client(self) -> aiogoogle.GoogleComputeClient: return self._compute_client - def user_credentials(self, credentials: Dict[str, bytes]) -> GCPUserCredentials: + def user_credentials(self, credentials: Dict[str, str]) -> GCPUserCredentials: return GCPUserCredentials(credentials) async def worker_access_token(self, session: httpx.ClientSession) -> Dict[str, str]: diff --git a/batch/batch/worker/credentials.py b/batch/batch/worker/credentials.py index cb909b88dac..bc61d50cb76 100644 --- a/batch/batch/worker/credentials.py +++ b/batch/batch/worker/credentials.py @@ -1,39 +1,26 @@ import abc -from typing import Dict class CloudUserCredentials(abc.ABC): @property - def secret_name(self) -> str: - raise NotImplementedError - - @property - def secret_data(self) -> Dict[str, bytes]: - raise NotImplementedError - - @property - def file_name(self) -> str: - raise NotImplementedError - - @property + @abc.abstractmethod def cloud_env_name(self) -> str: raise NotImplementedError @property - def hail_env_name(self) -> str: - raise NotImplementedError - - @property + @abc.abstractmethod def username(self) -> str: raise NotImplementedError @property + @abc.abstractmethod def password(self) -> str: raise NotImplementedError @property + @abc.abstractmethod def mount_path(self): - return f'/{self.secret_name}/{self.file_name}' + raise NotImplementedError @abc.abstractmethod def cloudfuse_credentials(self, fuse_config: dict) -> str: diff --git a/batch/batch/worker/worker.py b/batch/batch/worker/worker.py index 96a2c399f79..4337b9bcc66 100644 --- a/batch/batch/worker/worker.py +++ b/batch/batch/worker/worker.py @@ -1395,12 +1395,6 @@ def cloudfuse_tmp_path(self, bucket: str) -> str: def cloudfuse_credentials_path(self, bucket: str) -> str: return f'{self.scratch}/cloudfuse/{bucket}' - def credentials_host_dirname(self) -> str: - return f'{self.scratch}/{self.credentials.secret_name}' - - def credentials_host_file_path(self) -> str: - return f'{self.credentials_host_dirname()}/{self.credentials.file_name}' - @staticmethod def create( batch_id, diff --git a/batch/batch/worker/worker_api.py b/batch/batch/worker/worker_api.py index 2a459827a54..4aabdf70156 100644 --- a/batch/batch/worker/worker_api.py +++ b/batch/batch/worker/worker_api.py @@ -26,7 +26,7 @@ def get_cloud_async_fs(self) -> AsyncFS: raise NotImplementedError @abc.abstractmethod - def user_credentials(self, credentials: Dict[str, bytes]) -> CloudUserCredentials: + def user_credentials(self, credentials: Dict[str, str]) -> CloudUserCredentials: raise NotImplementedError @abc.abstractmethod From c5af9ba0032e6cfe4640d0df652d763dcbdf1e63 Mon Sep 17 00:00:00 2001 From: Dan King Date: Thu, 27 Apr 2023 09:23:07 -0400 Subject: [PATCH 05/26] [terraform] enable VPC Flow logs in all our US subnets (#12883) NB, this is a stacked PR. To see just these changes see [this commit](https://github.com/hail-is/hail/pull/12883/commits/ae51e0a9af12e4c89a44e7ce3235f3f665ff4830) --- [VPC Flow Logs](https://cloud.google.com/vpc/docs/flow-logs): > VPC Flow Logs records a sample of network flows sent from and received by VM instances, including > instances used as Google Kubernetes Engine nodes. These logs can be used for network monitoring, > forensics, real-time security analysis, and expense optimization I found the collection process the most elucidating part of the documentation. My summary of that process follows: 1. Packets are sampled on the network interface of a VM. Google claims an average sampling rate of 1/30. This rate reduces if the VM is under load. This rate is immutable to us. 2. Within an "aggregation interval", packets are aggregated into "records" which are keyed (my term) by source & destination. There are currently six choices for aggregation interval: 5s, 30s, 1m, 5m, 10m, and 15m. 3. Records are sampled. The sampling rate is a user configured floating point number (precision unclear) between 0 and 1. 4. Metadata is optionally added to the records. The metadata captures information about the source and destination VM such as project id, VM name, zone, region, GKE pod, GKE service, and geographic information of external parties. The user may elect to receive all metadata, no metadata, or a specific set of metadata fields. 5. The records are written to Google Cloud Logging. The pricing of VPC Flow Logs is described at the [network pricing page](https://cloud.google.com/vpc/network-pricing#network-telemetry). Notice that, if logs are only sent to Cloud Logging (not to BigQuery, Pub/Sub, or Cloud Storage): > If you store your logs in Cloud Logging, logs generation charges are waived, and only Logging charges apply. I believe in this phrase "logs generation charges" refers to *VPC Flow logs* generation charges. The Google Cloud Logging [pricing page](https://cloud.google.com/stackdriver/pricing#google-clouds-operations-suite-pricing) indicates that, after 50 GiB of free logs, the user is charged 0.50 USD per GiB of logs. Storage is free for thirty days and 0.01 USD per GiB for each additional day. We can calculate the cost of our logs as follows. Refer to the [definition of the record format](https://cloud.google.com/vpc/docs/flow-logs#record_format) for details. ```python3 ip_string = len("123.123.123.123") ip_connection = 4 + ip_string + ip_string + 4 + 4 date_time = len("1937-01-01T12:00:27.87+00:20") record_bytes = sum(( ip_connection, max(len('SRC'), len('DEST')), 8, 8, 8, date_time, date_time, )) assert record_bytes == 126 hours_per_month = 24 * 60 seconds_per_hour = 60 * 60 seconds_per_interval = 15 * 60 vms = 10000 sampling_rate = 0.5 connections_per_vm_per_aggregation_interval = 100 intervals_per_hour = seconds_per_hour / seconds_per_interval records_per_hour = intervals_per_hour * vms * connections_per_vm_per_aggregation_interval * sampling_rate bytes_per_hour = records_per_hour * record_bytes bytes_per_month = bytes_per_hour * hours_per_month GiB_per_month = bytes_per_month / 1024. / 1024 / 1024 USD_per_month = max(0, GiB_per_month - 50) * 0.5 print(GiB_per_month) print(USD_per_month) ``` This works out to 143 USD to run a 10,000 VM cluster 24 hours a day for 30 days. I suspect our average VM count in a month is closer to 10 which is within the free tier (340 MiB). I might be wrong abou the connections per vm per aggregation interval, but this is straightforward to monitor once we have the logs. For a sense of the cost landscape, these are all free: 1. 1000 VMs. 2. 500 VMs, with a sampling rate of 1. 3. 200 VMs, with a sampling rate of 1, with an interval of 5 minutes. 4. 10 VMs, with a sampling rate of 1, with an interval of 30 seconds. It's all linear, so if we need to halve the interval we can either change the sampling rate, reasses our expected number of VM-hours, or adjust the service fee accordingly. We can also assess the landscape of fees necessary to cover costs (ignoring the free 50 GiB): 1. 15 minute intervals, 0.5 sampling rate, 100 expected connections per vm per interval: 0.0000008 USD per core per hour. 2. 30 second intervals, 1.0 sampling rate, 100 expected connections per vm per interval: 0.00005 USD per core per hour. 2. 5 second intervals, 1.0 sampling rate, 100 expected connections per vm per interval: 0.0003 USD per core per hour. 2. 5 second intervals, 1.0 sampling rate, 1000 expected connections per vm per interval (1000 unique connections per second honestly seems to me quite remarkable performance): 0.003 USD per core per hour. ``` USD_per_core_per_hour = bytes_per_hour / vms / 1024. / 1024 / 1024 * 0.5 / 16 print(USD_per_core_per_hour) ``` --- # Conclusion I think we're safe to enable this with the parameters in this PR (15 minute intervals, 50% sampling). We can assess unknown parameters, like connections per vm, and get comfortable looking at these logs. Security constraints or observability demands may push us towards desiring more logs. If that occurs, we can assess the need for a new fee. Regardless, this fee appears to be small relative to the current cost of preemptible cores. --- infra/gcp-broad/main.tf | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/infra/gcp-broad/main.tf b/infra/gcp-broad/main.tf index 4d68d0d7d54..94faa34b4c6 100644 --- a/infra/gcp-broad/main.tf +++ b/infra/gcp-broad/main.tf @@ -92,6 +92,40 @@ resource "google_compute_subnetwork" "default_region" { ip_cidr_range = var.default_subnet_ip_cidr_range private_ip_google_access = true + log_config { + aggregation_interval = "INTERVAL_15_MIN" + flow_sampling = 0.5 + metadata = "EXCLUDE_ALL_METADATA" + } + + timeouts {} +} + +resource "google_compute_subnetwork" "us_nondefault_subnets" { + for_each = { + us-east1 = "10.142.0.0/20", + us-east4 = "10.150.0.0/20", + us-east5 = "10.202.0.0/20", + us-east7 = "10.196.0.0/20", + us-south1 = "10.206.0.0/20", + us-west1 = "10.138.0.0/20", + us-west2 = "10.168.0.0/20", + us-west3 = "10.180.0.0/20", + us-west4 = "10.182.0.0/20", + } + + name = "default" + region = each.key + network = google_compute_network.default.id + ip_cidr_range = each.value + private_ip_google_access = true + + log_config { + aggregation_interval = "INTERVAL_15_MIN" + flow_sampling = 0.5 + metadata = "EXCLUDE_ALL_METADATA" + } + timeouts {} } From 3427f331d471ccff191fd449f12251c6b8c30555 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Thu, 27 Apr 2023 16:05:45 -0400 Subject: [PATCH 06/26] [batch] Add timeouts to ensure that post_job_complete is reached (#12944) We always ensure that `cleanup` is run, but in order to ensure that `post_job_complete` is run we need to ensure we get through any computation in cleanup and `mark_complete` that could potentially hang. So we add timeouts to any yield points in those functions and broadly catch exceptions so we always continue in the cleanup/mark_complete process regardless of failure. --- batch/batch/worker/worker.py | 51 ++++++++++++++++++++++++------------ build.yaml | 26 ++++++++---------- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/batch/batch/worker/worker.py b/batch/batch/worker/worker.py index 4337b9bcc66..e5085f53baa 100644 --- a/batch/batch/worker/worker.py +++ b/batch/batch/worker/worker.py @@ -1544,13 +1544,19 @@ async def mark_complete(self, mjs_fut: asyncio.Task): if self.format_version.has_full_status_in_gcs(): assert self.worker.file_store - await retry_transient_errors( - self.worker.file_store.write_status_file, - self.batch_id, - self.job_id, - self.attempt_id, - json.dumps(full_status), - ) + try: + async with async_timeout.timeout(120): + await retry_transient_errors( + self.worker.file_store.write_status_file, + self.batch_id, + self.job_id, + self.attempt_id, + json.dumps(full_status), + ) + except asyncio.CancelledError: + raise + except Exception: + log.exception(f'Encountered error while writing status file for job {self.id}') if not self.deleted: self.task_manager.ensure_future(self.worker.post_job_complete(self, mjs_fut, full_status)) @@ -1890,12 +1896,13 @@ async def run(self): async def cleanup(self): if self.disk: try: - await self.disk.delete() - log.info(f'deleted disk {self.disk.name} for {self.id}') + async with async_timeout.timeout(300): + await self.disk.delete() + log.info(f'deleted disk {self.disk.name} for {self.id}') except asyncio.CancelledError: raise except Exception: - log.exception(f'while detaching and deleting disk {self.disk.name} for {self.id}') + log.exception(f'while detaching and deleting disk {self.disk.name} for job {self.id}') else: self.worker.data_disk_space_remaining.value += self.external_storage_in_gib @@ -1908,22 +1915,32 @@ async def cleanup(self): try: assert CLOUD_WORKER_API - await CLOUD_WORKER_API.unmount_cloudfuse(mount_path) - log.info(f'unmounted fuse blob storage {bucket} from {mount_path}') - config['mounted'] = False + async with async_timeout.timeout(120): + await CLOUD_WORKER_API.unmount_cloudfuse(mount_path) + log.info(f'unmounted fuse blob storage {bucket} from {mount_path}') + config['mounted'] = False except asyncio.CancelledError: raise except Exception: - log.exception(f'while unmounting fuse blob storage {bucket} from {mount_path}') + log.exception( + f'while unmounting fuse blob storage {bucket} from {mount_path} for job {self.id}' + ) - await check_shell(f'xfs_quota -x -c "limit -p bsoft=0 bhard=0 {self.project_id}" /host') + try: + async with async_timeout.timeout(120): + await check_shell(f'xfs_quota -x -c "limit -p bsoft=0 bhard=0 {self.project_id}" /host') + except asyncio.CancelledError: + raise + except Exception: + log.exception(f'while resetting xfs_quota project {self.project_id} for job {self.id}') try: - await blocking_to_async(self.pool, shutil.rmtree, self.scratch, ignore_errors=True) + async with async_timeout.timeout(120): + await blocking_to_async(self.pool, shutil.rmtree, self.scratch, ignore_errors=True) except asyncio.CancelledError: raise except Exception: - log.exception('while deleting volumes') + log.exception(f'while deleting scratch dir for job {self.id}') def get_container_log_path(self, container_name: str) -> str: return self.containers[container_name].log_path diff --git a/build.yaml b/build.yaml index 299c3891d96..8503c40a05e 100644 --- a/build.yaml +++ b/build.yaml @@ -599,17 +599,17 @@ steps: dependsOn: - merge_code - hail_ubuntu_image - - kind: buildImage2 - name: hailgenetics_vep_grch38_95_image - dockerFile: /io/repo/docker/hailgenetics/vep/grch38/95/Dockerfile - contextPath: /io/repo/docker/vep/ - publishAs: hailgenetics/vep-grch38-95 - inputs: - - from: /repo - to: /io/repo - dependsOn: - - merge_code - - hail_ubuntu_image + # - kind: buildImage2 + # name: hailgenetics_vep_grch38_95_image + # dockerFile: /io/repo/docker/hailgenetics/vep/grch38/95/Dockerfile + # contextPath: /io/repo/docker/vep/ + # publishAs: hailgenetics/vep-grch38-95 + # inputs: + # - from: /repo + # to: /io/repo + # dependsOn: + # - merge_code + # - hail_ubuntu_image - kind: buildImage2 name: monitoring_image dockerFile: /io/repo/monitoring/Dockerfile @@ -2341,7 +2341,6 @@ steps: export HAIL_TEST_RESOURCES_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/test/resources/" export HAIL_DOCTEST_DATA_DIR="{{ global.test_storage_uri }}/{{ upload_test_resources_to_blob_storage.token }}/doctest/data/" export HAIL_GENETICS_VEP_GRCH37_85_IMAGE={{ hailgenetics_vep_grch37_85_image.image }} - export HAIL_GENETICS_VEP_GRCH38_95_IMAGE={{ hailgenetics_vep_grch38_95_image.image }} export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json if [[ "$HAIL_CLOUD" = "gcp" ]] @@ -2412,7 +2411,6 @@ steps: - upload_test_resources_to_blob_storage - build_hail_jar_and_wheel_only - hailgenetics_vep_grch37_85_image - - hailgenetics_vep_grch38_95_image - kind: buildImage2 name: netcat_ubuntu_image publishAs: netcat @@ -3213,7 +3211,6 @@ steps: docker://{{ hailgenetics_hail_image.image }} \ docker://{{ hailgenetics_hailtop_image.image }} \ docker://{{ hailgenetics_vep_grch37_85_image.image }} \ - docker://{{ hailgenetics_vep_grch38_95_image.image }} \ /io/wheel-for-azure/hail-*-py3-none-any.whl \ /io/www.tar.gz inputs: @@ -3259,7 +3256,6 @@ steps: - hailgenetics_hail_image - hailgenetics_hailtop_image - hailgenetics_vep_grch37_85_image - - hailgenetics_vep_grch38_95_image - build_wheel_for_azure - make_docs clouds: From bcc7cb1a4083e469671d892386a758c66ac22865 Mon Sep 17 00:00:00 2001 From: jigold Date: Thu, 27 Apr 2023 18:14:29 -0400 Subject: [PATCH 07/26] [batch] Fix bad insert attempt resources trigger (#12942) There was an extra check in the where statement for the token matching on the jobs billing table. The jobs billing table is the only one not parameterized with a token. --- batch/sql/estimated-current.sql | 12 +- ...mitigate-bad-attempt-resources-trigger.sql | 107 ++++++++++++++++++ build.yaml | 3 + 3 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 batch/sql/mitigate-bad-attempt-resources-trigger.sql diff --git a/batch/sql/estimated-current.sql b/batch/sql/estimated-current.sql index 95afc2acb50..fce5bb24646 100644 --- a/batch/sql/estimated-current.sql +++ b/batch/sql/estimated-current.sql @@ -836,7 +836,8 @@ BEGIN SELECT migrated INTO bp_user_resources_migrated FROM aggregated_billing_project_user_resources_v2 - WHERE billing_project = cur_billing_project AND user = cur_user AND resource_id = NEW.resource_id AND token = rand_token; + WHERE billing_project = cur_billing_project AND user = cur_user AND resource_id = NEW.resource_id AND token = rand_token + FOR UPDATE; IF bp_user_resources_migrated THEN INSERT INTO aggregated_billing_project_user_resources_v3 (billing_project, user, resource_id, token, `usage`) @@ -852,7 +853,8 @@ BEGIN SELECT migrated INTO batch_resources_migrated FROM aggregated_batch_resources_v2 - WHERE batch_id = NEW.batch_id AND resource_id = NEW.resource_id AND token = rand_token; + WHERE batch_id = NEW.batch_id AND resource_id = NEW.resource_id AND token = rand_token + FOR UPDATE; IF batch_resources_migrated THEN INSERT INTO aggregated_batch_resources_v3 (batch_id, resource_id, token, `usage`) @@ -868,7 +870,8 @@ BEGIN SELECT migrated INTO job_resources_migrated FROM aggregated_job_resources_v2 - WHERE batch_id = NEW.batch_id AND job_id = NEW.job_id AND resource_id = NEW.resource_id AND token = rand_token; + WHERE batch_id = NEW.batch_id AND job_id = NEW.job_id AND resource_id = NEW.resource_id + FOR UPDATE; IF job_resources_migrated THEN INSERT INTO aggregated_job_resources_v3 (batch_id, job_id, resource_id, `usage`) @@ -885,7 +888,8 @@ BEGIN SELECT migrated INTO bp_user_resources_by_date_migrated FROM aggregated_billing_project_user_resources_by_date_v2 WHERE billing_date = cur_billing_date AND billing_project = cur_billing_project AND user = cur_user - AND resource_id = NEW.resource_id AND token = rand_token; + AND resource_id = NEW.resource_id AND token = rand_token + FOR UPDATE; IF bp_user_resources_by_date_migrated THEN INSERT INTO aggregated_billing_project_user_resources_by_date_v3 (billing_date, billing_project, user, resource_id, token, `usage`) diff --git a/batch/sql/mitigate-bad-attempt-resources-trigger.sql b/batch/sql/mitigate-bad-attempt-resources-trigger.sql new file mode 100644 index 00000000000..b5e392e7917 --- /dev/null +++ b/batch/sql/mitigate-bad-attempt-resources-trigger.sql @@ -0,0 +1,107 @@ +DELIMITER $$ + +DROP TRIGGER IF EXISTS attempt_resources_after_insert $$ +CREATE TRIGGER attempt_resources_after_insert AFTER INSERT ON attempt_resources +FOR EACH ROW +BEGIN + DECLARE cur_start_time BIGINT; + DECLARE cur_rollup_time BIGINT; + DECLARE cur_billing_project VARCHAR(100); + DECLARE cur_user VARCHAR(100); + DECLARE msec_diff_rollup BIGINT; + DECLARE cur_n_tokens INT; + DECLARE rand_token INT; + DECLARE cur_billing_date DATE; + DECLARE bp_user_resources_migrated BOOLEAN DEFAULT FALSE; + DECLARE bp_user_resources_by_date_migrated BOOLEAN DEFAULT FALSE; + DECLARE batch_resources_migrated BOOLEAN DEFAULT FALSE; + DECLARE job_resources_migrated BOOLEAN DEFAULT FALSE; + + SELECT billing_project, user INTO cur_billing_project, cur_user + FROM batches WHERE id = NEW.batch_id; + + SELECT n_tokens INTO cur_n_tokens FROM globals LOCK IN SHARE MODE; + SET rand_token = FLOOR(RAND() * cur_n_tokens); + + SELECT start_time, rollup_time INTO cur_start_time, cur_rollup_time + FROM attempts + WHERE batch_id = NEW.batch_id AND job_id = NEW.job_id AND attempt_id = NEW.attempt_id + LOCK IN SHARE MODE; + + SET msec_diff_rollup = GREATEST(COALESCE(cur_rollup_time - cur_start_time, 0), 0); + + SET cur_billing_date = CAST(UTC_DATE() AS DATE); + + IF msec_diff_rollup != 0 THEN + INSERT INTO aggregated_billing_project_user_resources_v2 (billing_project, user, resource_id, token, `usage`) + VALUES (cur_billing_project, cur_user, NEW.resource_id, rand_token, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + + SELECT migrated INTO bp_user_resources_migrated + FROM aggregated_billing_project_user_resources_v2 + WHERE billing_project = cur_billing_project AND user = cur_user AND resource_id = NEW.resource_id AND token = rand_token + FOR UPDATE; + + IF bp_user_resources_migrated THEN + INSERT INTO aggregated_billing_project_user_resources_v3 (billing_project, user, resource_id, token, `usage`) + VALUES (cur_billing_project, cur_user, NEW.deduped_resource_id, rand_token, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + END IF; + + INSERT INTO aggregated_batch_resources_v2 (batch_id, resource_id, token, `usage`) + VALUES (NEW.batch_id, NEW.resource_id, rand_token, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + + SELECT migrated INTO batch_resources_migrated + FROM aggregated_batch_resources_v2 + WHERE batch_id = NEW.batch_id AND resource_id = NEW.resource_id AND token = rand_token + FOR UPDATE; + + IF batch_resources_migrated THEN + INSERT INTO aggregated_batch_resources_v3 (batch_id, resource_id, token, `usage`) + VALUES (NEW.batch_id, NEW.deduped_resource_id, rand_token, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + END IF; + + INSERT INTO aggregated_job_resources_v2 (batch_id, job_id, resource_id, `usage`) + VALUES (NEW.batch_id, NEW.job_id, NEW.resource_id, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + + SELECT migrated INTO job_resources_migrated + FROM aggregated_job_resources_v2 + WHERE batch_id = NEW.batch_id AND job_id = NEW.job_id AND resource_id = NEW.resource_id + FOR UPDATE; + + IF job_resources_migrated THEN + INSERT INTO aggregated_job_resources_v3 (batch_id, job_id, resource_id, `usage`) + VALUES (NEW.batch_id, NEW.job_id, NEW.deduped_resource_id, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + END IF; + + INSERT INTO aggregated_billing_project_user_resources_by_date_v2 (billing_date, billing_project, user, resource_id, token, `usage`) + VALUES (cur_billing_date, cur_billing_project, cur_user, NEW.resource_id, rand_token, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + + SELECT migrated INTO bp_user_resources_by_date_migrated + FROM aggregated_billing_project_user_resources_by_date_v2 + WHERE billing_date = cur_billing_date AND billing_project = cur_billing_project AND user = cur_user + AND resource_id = NEW.resource_id AND token = rand_token + FOR UPDATE; + + IF bp_user_resources_by_date_migrated THEN + INSERT INTO aggregated_billing_project_user_resources_by_date_v3 (billing_date, billing_project, user, resource_id, token, `usage`) + VALUES (cur_billing_date, cur_billing_project, cur_user, NEW.deduped_resource_id, rand_token, NEW.quantity * msec_diff_rollup) + ON DUPLICATE KEY UPDATE + `usage` = `usage` + NEW.quantity * msec_diff_rollup; + END IF; + END IF; +END $$ + +DELIMITER ; diff --git a/build.yaml b/build.yaml index 8503c40a05e..9fc4c8e6ab8 100644 --- a/build.yaml +++ b/build.yaml @@ -2174,6 +2174,9 @@ steps: - name: dedup-attempt-resources script: /io/sql/dedup_attempt_resources.py online: true + - name: mitigate-bad-attempt-resources-trigger + script: /io/sql/mitigate-bad-attempt-resources-trigger.sql + online: true inputs: - from: /repo/batch/sql to: /io/sql From 13898748e6f2e9500c79b5e8d6e0fc0e55188fac Mon Sep 17 00:00:00 2001 From: Tim Poterba Date: Thu, 27 Apr 2023 20:53:21 -0400 Subject: [PATCH 08/26] [FS] Fix bug in GoogleStorageFS seek (#12945) --- hail/src/main/scala/is/hail/io/fs/FS.scala | 1 + .../is/hail/fs/gs/GoogleStorageFSSuite.scala | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/hail/src/main/scala/is/hail/io/fs/FS.scala b/hail/src/main/scala/is/hail/io/fs/FS.scala index a2fae8788bf..18f8319088d 100644 --- a/hail/src/main/scala/is/hail/io/fs/FS.scala +++ b/hail/src/main/scala/is/hail/io/fs/FS.scala @@ -177,6 +177,7 @@ abstract class FSSeekableInputStream extends InputStream with Seekable { protected def physicalSeek(newPos: Long): Unit def seek(newPos: Long): Unit = { + eof = false val distance = newPos - pos val bufferSeekPosition = bb.position() + distance if (bufferSeekPosition >= 0 && bufferSeekPosition < bb.limit()) { diff --git a/hail/src/test/scala/is/hail/fs/gs/GoogleStorageFSSuite.scala b/hail/src/test/scala/is/hail/fs/gs/GoogleStorageFSSuite.scala index 9cab85a9dde..49600b71bc0 100644 --- a/hail/src/test/scala/is/hail/fs/gs/GoogleStorageFSSuite.scala +++ b/hail/src/test/scala/is/hail/fs/gs/GoogleStorageFSSuite.scala @@ -3,6 +3,7 @@ package is.hail.fs.gs import java.io.FileInputStream import is.hail.fs.FSSuite import is.hail.io.fs.GoogleStorageFS +import is.hail.utils._ import org.apache.commons.io.IOUtils import org.scalatest.testng.TestNGSuite import org.testng.annotations.{BeforeClass, Test} @@ -61,4 +62,24 @@ class GoogleStorageFSSuite extends TestNGSuite with FSSuite { fs.delete(prefix, recursive = true) assert(!fs.exists(prefix), s"files not deleted:\n${ fs.listStatus(prefix).map(_.getPath).mkString("\n") }") } + + @Test def testSeekAfterEOF(): Unit = { + val prefix = s"$hail_test_storage_uri/google-storage-fs-suite/delete-many-files/${ java.util.UUID.randomUUID() }" + val p = s"$prefix/seek_file" + using(fs.createCachedNoCompression(p)) { os => + os.write(1.toByte) + os.write(2.toByte) + os.write(3.toByte) + os.write(4.toByte) + } + + using(fs.openNoCompression(p)) { is => + assert(is.read() == 1.toByte) + is.seek(3) + assert(is.read() == 4.toByte) + assert(is.read() == (-1).toByte) + is.seek(0) + assert(is.read() == 1.toByte) + } + } } From 948b1d96278823a0e7c8a07adba2b2dfb95334de Mon Sep 17 00:00:00 2001 From: Dan King Date: Thu, 27 Apr 2023 23:02:18 -0400 Subject: [PATCH 09/26] [query] fix & improve pprint for hl.Struct (#12901) CHANGELOG: `hl.Struct` now has a correct and useful implementation of pprint. For structs with keys that were not identifiers, we produced incorrect `repr` output. For `pprint`, we just `pprint`'ed a dictionary (so you cannot even tell that the object was an `hl.Struct`). This PR: 1. Fixes `hl.Struct.__str__` to use the kwargs or dictionary representation based on whether the keys are Python identifiers. 2. Teaches `StructPrettyPrinter` to first try to `repr` the struct (this is what the default pretty printer does) 3. Teaches `StructPrettyPrinter` to properly pretty print a struct as an `hl.Struct` preferring the kwarg representation when appropriate. 4. Teaches `_same` to use pretty printing when showing differing records. --- hail/python/hail/table.py | 5 +-- hail/python/hail/utils/struct.py | 58 +++++++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/hail/python/hail/table.py b/hail/python/hail/table.py index c84b37fb5b0..3fbb203d9c6 100644 --- a/hail/python/hail/table.py +++ b/hail/python/hail/table.py @@ -3,6 +3,7 @@ import pandas import numpy as np import pyspark +import pprint from typing import Optional, Dict, Callable, Sequence, Union from hail.expr.expressions import Expression, StructExpression, \ @@ -3669,7 +3670,7 @@ def _same(self, other, tolerance=1e-6, absolute=False, reorder_fields=False): if not hl.eval(_values_similar(t[left_global_value], t[right_global_value], tolerance, absolute)): g = hl.eval(t.globals) - print(f'Table._same: globals differ: {g[left_global_value]}, {g[right_global_value]}') + print(f'Table._same: globals differ:\n{pprint.pformat(g[left_global_value])}\n{pprint.pformat(g[right_global_value])}') return False if not t.all(hl.is_defined(t[left_value]) & hl.is_defined(t[right_value]) @@ -3678,7 +3679,7 @@ def _same(self, other, tolerance=1e-6, absolute=False, reorder_fields=False): t = t.filter(~ _values_similar(t[left_value], t[right_value], tolerance, absolute)) bad_rows = t.take(10) for r in bad_rows: - print(f' Row mismatch at key={r._key}:\n L: {r[left_value]}\n R: {r[right_value]}') + print(f' Row mismatch at key={r._key}:\n Left:\n{pprint.pformat(r[left_value])}\n Right:\n{pprint.pformat(r[right_value])}') return False return True diff --git a/hail/python/hail/utils/struct.py b/hail/python/hail/utils/struct.py index 71007b9bd8b..356edd025e6 100644 --- a/hail/python/hail/utils/struct.py +++ b/hail/python/hail/utils/struct.py @@ -81,7 +81,17 @@ def __repr__(self): return str(self) def __str__(self): - return 'Struct({})'.format(', '.join('{}={}'.format(k, repr(v)) for k, v in self._fields.items())) + if all(k.isidentifier() for k in self._fields): + return ( + 'Struct(' + + ', '.join(f'{k}={repr(v)}' for k, v in self._fields.items()) + + ')' + ) + return ( + 'Struct(**{' + + ', '.join(f'{repr(k)}: {repr(v)}' for k, v in self._fields.items()) + + '})' + ) def __eq__(self, other): return isinstance(other, Struct) and self._fields == other._fields @@ -241,10 +251,50 @@ def to_dict(struct): class StructPrettyPrinter(pprint.PrettyPrinter): - def _format(self, obj, *args, **kwargs): + def _format(self, obj, stream, indent, allowance, context, level, *args, **kwargs): if isinstance(obj, Struct): - obj = to_dict(obj) - return _old_printer._format(self, obj, *args, **kwargs) + rep = self._repr(obj, context, level) + max_width = self._width - indent - allowance + if len(rep) <= max_width: + stream.write(rep) + return + + stream.write('Struct(') + indent += len('Struct(') + if all(k.isidentifier() for k in obj): + n = len(obj.items()) + for i, (k, v) in enumerate(obj.items()): + is_first = i == 0 + is_last = i == n - 1 + + if not is_first: + stream.write(' ' * indent) + stream.write(k) + stream.write('=') + this_indent = indent + len(k) + len('=') + self._format(v, stream, this_indent, allowance, context, level, *args, **kwargs) + if not is_last: + stream.write(',\n') + else: + stream.write('**{') + indent += len('**{') + n = len(obj.items()) + for i, (k, v) in enumerate(obj.items()): + is_first = i == 0 + is_last = i == n - 1 + + if not is_first: + stream.write(' ' * indent) + stream.write(repr(k)) + stream.write(': ') + this_indent = indent + len(repr(k)) + len(': ') + self._format(v, stream, this_indent, allowance, context, level, *args, **kwargs) + if not is_last: + stream.write(',\n') + stream.write('}') + stream.write(')') + else: + _old_printer._format(self, obj, stream, indent, allowance, context, level, *args, **kwargs) pprint.PrettyPrinter = StructPrettyPrinter # monkey-patch pprint From e9b2ee60b3dbd139aa4e3eeb642042259fff76b7 Mon Sep 17 00:00:00 2001 From: Dan King Date: Fri, 28 Apr 2023 12:15:06 -0400 Subject: [PATCH 10/26] [is_transient_error] retry once ConnectionRefusedError (#12947) --- hail/python/hailtop/utils/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hail/python/hailtop/utils/utils.py b/hail/python/hailtop/utils/utils.py index 9fb6224c0e3..f63f61a5d48 100644 --- a/hail/python/hailtop/utils/utils.py +++ b/hail/python/hailtop/utils/utils.py @@ -592,6 +592,8 @@ def is_retry_once_error(e): return e.status == 400 and any(msg in e.body for msg in RETRY_ONCE_BAD_REQUEST_ERROR_MESSAGES) if isinstance(e, ConnectionResetError): return True + if isinstance(e, ConnectionRefusedError): + return True if e.__cause__ is not None: return is_transient_error(e.__cause__) return False From e0dbb8b544259cdfd88810fa1111d59e46cf7279 Mon Sep 17 00:00:00 2001 From: jigold Date: Fri, 28 Apr 2023 16:33:53 -0400 Subject: [PATCH 11/26] [batch] Turn off support for cloudfuse (#12949) This is a mitigation for turning off cloudfuse until we understand why the unmount is not working properly. --- batch/batch/front_end/front_end.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/batch/batch/front_end/front_end.py b/batch/batch/front_end/front_end.py index 7e8747a2dc7..583646f0012 100644 --- a/batch/batch/front_end/front_end.py +++ b/batch/batch/front_end/front_end.py @@ -832,6 +832,7 @@ async def _create_jobs(userdata: dict, job_specs: dict, batch_id: int, update_id db: Database = app['db'] file_store: FileStore = app['file_store'] user = userdata['username'] + is_developer = userdata['is_developer'] # restrict to what's necessary; in particular, drop the session # which is sensitive @@ -1063,6 +1064,10 @@ async def _create_jobs(userdata: dict, job_specs: dict, batch_id: int, update_id if cloud == 'azure' and all(envvar['name'] != 'AZURE_APPLICATION_CREDENTIALS' for envvar in spec['env']): spec['env'].append({'name': 'AZURE_APPLICATION_CREDENTIALS', 'value': '/gsa-key/key.json'}) + cloudfuse = spec.get('gcsfuse') or spec.get('cloudfuse') + if not is_developer and user not in ('ci', 'test', 'test-dev') and cloudfuse is not None and len(cloudfuse) > 0: + raise web.HTTPBadRequest(reason='cloudfuse requests are temporarily not supported.') + if spec.get('mount_tokens', False): secrets.append( { From 1940547d35ddddb084ad52684e36153c1e03a331 Mon Sep 17 00:00:00 2001 From: Dan King Date: Mon, 1 May 2023 13:36:34 -0400 Subject: [PATCH 12/26] [qob] fix job logs (#12941) Spark 3.3.0 uses log4j2. Note the "2". If you use the log4j1 programmatic reconfiguration system, you will break log4j2 for you and everyone else. The only way to recover from such a breakage is to use the log4j2 programmatic reconfiguration system. Changes in this PR: 1. Include JVM output in error logs when the JVM crashes. This should help debugging of JVM crashing in production until the JVM logs are shown on a per-worker page. 2. JVMEntryway is now a real gradle project. I need to compile against log4j, and I didn't want to do that by hand with `javac`. Ignore gradlew, gradlew.bat, and gradle/wrapper, they're programmatically generated by gradle. 3. Add logging to JVMEntryway. JVMEntryway now logs its arguments into the QoB job log. I also log exceptions from the main thread or the cancel thread into the job log. We also flush the logs after the main thread completes, the cancel thread completes, and when the try-catch exits. This should ensure that regardless of what goes wrong (even if both threads fail to start) we at least see the arguments that the JVMEntryway received. 4. Use log4j2 programmatic reconfiguration after every job. This restores log4j2 to well enough working order that, *if you do not try to reconfigure it using log4j1 programmatic configuration*, logs will work. All old versions of Hail use log4j1 programmatic configuration. As a result, **all old versions of Hail will still have no logs**. However, new versions of Hail will log correctly even if an old version of Hail used the JVM before it. 5. `QoBAppender`. This is how we always should have done logging. A custom appender which we can flush and then redirect to a new file at our whim. I followed the log4j2 best practices for creating a new appender. All these annotations, factory methods, and managers are The Right Way, for better or worse. If we ever ban old versions of Hail from the cluster, then we can also eliminate the log4j2 reconfiguration. New versions of Hail work fine without any runtime log configuration (thanks to `QoBAppender`). I would like to eliminate reconfiguration because log4j2 reconfiguration leaves around oprhaned appenders and appender managers. Maybe I'm implementing the Appender or Appender Manager interfaces wrong, but I've read over that code a bunch of times and I cannot sort out what I am missing. --- Makefile | 10 +- batch/Dockerfile.worker | 3 +- batch/batch/worker/worker.py | 33 +++- batch/jvm-entryway/.gitignore | 2 + batch/jvm-entryway/build.gradle | 62 ++++++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 5 + batch/jvm-entryway/gradlew | 185 ++++++++++++++++++ batch/jvm-entryway/gradlew.bat | 89 +++++++++ .../src/main/java/is/hail/JVMEntryway.java | 31 ++- .../src/main/resources/log4j2.properties | 13 ++ .../src/main/scala/is/hail/QoBAppender.scala | 83 ++++++++ build.yaml | 29 ++- .../scala/is/hail/backend/service/Main.scala | 18 -- 14 files changed, 508 insertions(+), 55 deletions(-) create mode 100644 batch/jvm-entryway/.gitignore create mode 100644 batch/jvm-entryway/build.gradle create mode 100644 batch/jvm-entryway/gradle/wrapper/gradle-wrapper.jar create mode 100644 batch/jvm-entryway/gradle/wrapper/gradle-wrapper.properties create mode 100755 batch/jvm-entryway/gradlew create mode 100644 batch/jvm-entryway/gradlew.bat rename batch/{ => jvm-entryway}/src/main/java/is/hail/JVMEntryway.java (83%) create mode 100644 batch/jvm-entryway/src/main/resources/log4j2.properties create mode 100644 batch/jvm-entryway/src/main/scala/is/hail/QoBAppender.scala diff --git a/Makefile b/Makefile index a49e2a2748b..8c954c09509 100644 --- a/Makefile +++ b/Makefile @@ -165,14 +165,10 @@ hail-buildkit-image: ci/buildkit/Dockerfile ./docker-build.sh ci buildkit/Dockerfile.out $(HAIL_BUILDKIT_IMAGE) echo $(HAIL_BUILDKIT_IMAGE) > $@ -batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar: - mkdir -p batch/jars - cd batch/jars && curl -LO https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar +batch/jvm-entryway/build/libs/jvm-entryway.jar: $(shell git ls-files batch/jvm-entryway) + cd batch/jvm-entryway && ./gradlew shadowJar -batch/src/main/java/is/hail/JVMEntryway.class: batch/src/main/java/is/hail/JVMEntryway.java batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar - javac -cp batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar $< - -batch-worker-image: batch/src/main/java/is/hail/JVMEntryway.class $(SERVICES_IMAGE_DEPS) $(shell git ls-files batch) +batch-worker-image: batch/jvm-entryway/build/libs/jvm-entryway.jar $(SERVICES_IMAGE_DEPS) $(shell git ls-files batch) $(eval BATCH_WORKER_IMAGE := $(DOCKER_PREFIX)/batch-worker:$(TOKEN)) python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"},"global":{"cloud":"$(CLOUD)"}}' batch/Dockerfile.worker batch/Dockerfile.worker.out ./docker-build.sh . batch/Dockerfile.worker.out $(BATCH_WORKER_IMAGE) diff --git a/batch/Dockerfile.worker b/batch/Dockerfile.worker index 76cfb791972..4b58071e37e 100644 --- a/batch/Dockerfile.worker +++ b/batch/Dockerfile.worker @@ -71,6 +71,5 @@ COPY batch/batch /batch/batch/ RUN hail-pip-install /hailtop /gear /batch -COPY batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar /jvm-entryway/ -COPY batch/src/main/java/is /jvm-entryway/is +COPY batch/jvm-entryway/build/libs/jvm-entryway.jar /jvm-entryway/ COPY letsencrypt/subdomains.txt / diff --git a/batch/batch/worker/worker.py b/batch/batch/worker/worker.py index e5085f53baa..935e36a422b 100644 --- a/batch/batch/worker/worker.py +++ b/batch/batch/worker/worker.py @@ -2338,7 +2338,7 @@ async def create_and_start( 'java', f'-Xmx{heap_memory_mib}M', '-cp', - f'/jvm-entryway:/jvm-entryway/junixsocket-selftest-2.3.3-jar-with-dependencies.jar:{JVM.SPARK_HOME}/jars/*', + f'/jvm-entryway/jvm-entryway.jar:{JVM.SPARK_HOME}/jars/*', 'is.hail.JVMEntryway', socket_file, ] @@ -2638,14 +2638,29 @@ async def execute(self, classpath: str, scratch_dir: str, log_file: str, jar_url elif message == JVM.FINISH_USER_EXCEPTION: exception = await read_str(reader) raise JVMUserError(exception) - elif message == JVM.FINISH_ENTRYWAY_EXCEPTION: - log.warning(f'{self}: entryway exception encountered (interrupted: {wait_for_interrupt.done()})') - exception = await read_str(reader) - raise ValueError(exception) - elif message == JVM.FINISH_JVM_EOS: - assert eos_exception is not None - log.warning(f'{self}: unexpected end of stream in jvm (interrupted: {wait_for_interrupt.done()})') - raise ValueError('unexpected end of stream in jvm') from eos_exception + else: + jvm_output = '' + if os.path.exists(self.container.container.log_path): + jvm_output = (await self.fs.read(self.container.container.log_path)).decode('utf-8') + + if message == JVM.FINISH_ENTRYWAY_EXCEPTION: + log.warning( + f'{self}: entryway exception encountered (interrupted: {wait_for_interrupt.done()})\nJVM Output:\n\n{jvm_output}' + ) + exception = await read_str(reader) + raise ValueError(exception) + if message == JVM.FINISH_JVM_EOS: + assert eos_exception is not None + log.warning( + f'{self}: unexpected end of stream in jvm (interrupted: {wait_for_interrupt.done()})\nJVM Output:\n\n{jvm_output}' + ) + raise ValueError( + # Do not include the JVM log in the exception as this is sent to the user and + # the JVM log might inadvetantly contain sensitive information. + 'unexpected end of stream in jvm' + ) from eos_exception + log.exception(f'{self}: unexpected message type: {message}\nJVM Output:\n\n{jvm_output}') + raise ValueError(f'{self}: unexpected message type: {message}') class Worker: diff --git a/batch/jvm-entryway/.gitignore b/batch/jvm-entryway/.gitignore new file mode 100644 index 00000000000..69dd2d5b2c0 --- /dev/null +++ b/batch/jvm-entryway/.gitignore @@ -0,0 +1,2 @@ +.bloop +.metals diff --git a/batch/jvm-entryway/build.gradle b/batch/jvm-entryway/build.gradle new file mode 100644 index 00000000000..c63e6bce58b --- /dev/null +++ b/batch/jvm-entryway/build.gradle @@ -0,0 +1,62 @@ +buildscript { + repositories { + mavenCentral() + } +} + +plugins { + id "application" + id 'java' + id 'scala' + id 'com.github.johnrengelman.shadow' version '8.1.1' +} + +repositories { + mavenCentral() + maven { url "https://repository.cloudera.com/artifactory/cloudera-repos/" } +} + +project.ext { + sparkVersion = System.getProperty("spark.version", "3.3.0") + scalaVersion = System.getProperty("scala.version", "2.12.13") +} + +sourceSets { + main { + scala { + // compile java and scala together so they can interdepend + srcDirs = ['src/main/scala', 'src/main/java'] + } + java { + srcDirs = [] + } + } +} + +dependencies { + implementation 'com.kohlschutter.junixsocket:junixsocket-core:2.6.2' + compileOnly 'org.scala-lang:scala-library:' + scalaVersion + compileOnly 'org.scala-lang:scala-reflect:' + scalaVersion + compileOnly('org.apache.spark:spark-core_2.12:' + sparkVersion) { + exclude module: 'hadoop-client' + } +} + +jar { + manifest { + attributes 'Main-Class': application.mainClass + } +} + +shadowJar { + archiveBaseName.set('jvm-entryway') + archiveClassifier.set('') + archiveVersion.set('') +} + +application { + mainClassName = "is.hail.JVMEntryway" + // these can help debug log4j + // applicationDefaultJvmArgs = ["-Dlog4j.debug"] + // applicationDefaultJvmArgs = ["-Dlog4j2.debug"] +} diff --git a/batch/jvm-entryway/gradle/wrapper/gradle-wrapper.jar b/batch/jvm-entryway/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..e708b1c023ec8b20f512888fe07c5bd3ff77bb8f GIT binary patch literal 59203 zcma&O1CT9Y(k9%tZQHhO+qUh#ZQHhO+qmuS+qP|E@9xZO?0h@l{(r>DQ>P;GjjD{w zH}lENr;dU&FbEU?00aa80D$0M0RRB{U*7-#kbjS|qAG&4l5%47zyJ#WrfA#1$1Ctx zf&Z_d{GW=lf^w2#qRJ|CvSJUi(^E3iv~=^Z(zH}F)3Z%V3`@+rNB7gTVU{Bb~90p|f+0(v;nz01EG7yDMX9@S~__vVgv%rS$+?IH+oZ03D5zYrv|^ zC1J)SruYHmCki$jLBlTaE5&dFG9-kq3!^i>^UQL`%gn6)jz54$WDmeYdsBE9;PqZ_ zoGd=P4+|(-u4U1dbAVQrFWoNgNd;0nrghPFbQrJctO>nwDdI`Q^i0XJDUYm|T|RWc zZ3^Qgo_Qk$%Fvjj-G}1NB#ZJqIkh;kX%V{THPqOyiq)d)0+(r9o(qKlSp*hmK#iIY zA^)Vr$-Hz<#SF=0@tL@;dCQsm`V9s1vYNq}K1B)!XSK?=I1)tX+bUV52$YQu*0%fnWEukW>mxkz+%3-S!oguE8u#MGzST8_Dy^#U?fA@S#K$S@9msUiX!gd_ow>08w5)nX{-KxqMOo7d?k2&?Vf z&diGDtZr(0cwPe9z9FAUSD9KC)7(n^lMWuayCfxzy8EZsns%OEblHFSzP=cL6}?J| z0U$H!4S_TVjj<`6dy^2j`V`)mC;cB%* z8{>_%E1^FH!*{>4a7*C1v>~1*@TMcLK{7nEQ!_igZC}ikJ$*<$yHy>7)oy79A~#xE zWavoJOIOC$5b6*q*F_qN1>2#MY)AXVyr$6x4b=$x^*aqF*L?vmj>Mgv+|ITnw_BoW zO?jwHvNy^prH{9$rrik1#fhyU^MpFqF2fYEt(;4`Q&XWOGDH8k6M=%@fics4ajI;st# zCU^r1CK&|jzUhRMv;+W~6N;u<;#DI6cCw-otsc@IsN3MoSD^O`eNflIoR~l4*&-%RBYk@gb^|-JXs&~KuSEmMxB}xSb z@K76cXD=Y|=I&SNC2E+>Zg?R6E%DGCH5J1nU!A|@eX9oS(WPaMm==k2s_ueCqdZw| z&hqHp)47`c{BgwgvY2{xz%OIkY1xDwkw!<0veB#yF4ZKJyabhyyVS`gZepcFIk%e2 zTcrmt2@-8`7i-@5Nz>oQWFuMC_KlroCl(PLSodswHqJ3fn<;gxg9=}~3x_L3P`9Sn zChIf}8vCHvTriz~T2~FamRi?rh?>3bX1j}%bLH+uFX+p&+^aXbOK7clZxdU~6Uxgy z8R=obwO4dL%pmVo*Ktf=lH6hnlz_5k3cG;m8lgaPp~?eD!Yn2kf)tU6PF{kLyn|oI@eQ`F z3IF7~Blqg8-uwUuWZScRKn%c2_}dXB6Dx_&xR*n9M9LXasJhtZdr$vBY!rP{c@=)& z#!?L$2UrkvClwQO>U*fSMs67oSj2mxiJ$t;E|>q%Kh_GzzWWO&3;ufU%2z%ucBU8H z3WIwr$n)cfCXR&>tyB7BcSInK>=ByZA%;cVEJhcg<#6N{aZC4>K41XF>ZgjG`z_u& zGY?;Ad?-sgiOnI`oppF1o1Gurqbi*;#x2>+SSV6|1^G@ooVy@fg?wyf@0Y!UZ4!}nGuLeC^l)6pwkh|oRY`s1Pm$>zZ3u-83T|9 zGaKJIV3_x+u1>cRibsaJpJqhcm%?0-L;2 zitBrdRxNmb0OO2J%Y&Ym(6*`_P3&&5Bw157{o7LFguvxC$4&zTy#U=W*l&(Q2MNO} zfaUwYm{XtILD$3864IA_nn34oVa_g^FRuHL5wdUd)+W-p-iWCKe8m_cMHk+=? zeKX)M?Dt(|{r5t7IenkAXo%&EXIb-i^w+0CX0D=xApC=|Xy(`xy+QG^UyFe z+#J6h_&T5i#sV)hj3D4WN%z;2+jJcZxcI3*CHXGmOF3^)JD5j&wfX)e?-|V0GPuA+ zQFot%aEqGNJJHn$!_}#PaAvQ^{3-Ye7b}rWwrUmX53(|~i0v{}G_sI9uDch_brX&6 zWl5Ndj-AYg(W9CGfQf<6!YmY>Ey)+uYd_JNXH=>|`OH-CDCmcH(0%iD_aLlNHKH z7bcW-^5+QV$jK?R*)wZ>r9t}loM@XN&M-Pw=F#xn(;u3!(3SXXY^@=aoj70;_=QE9 zGghsG3ekq#N||u{4We_25U=y#T*S{4I{++Ku)> zQ!DZW;pVcn>b;&g2;YE#+V`v*Bl&Y-i@X6D*OpNA{G@JAXho&aOk(_j^weW{#3X5Y z%$q_wpb07EYPdmyH(1^09i$ca{O<}7) zRWncXdSPgBE%BM#by!E>tdnc$8RwUJg1*x($6$}ae$e9Knj8gvVZe#bLi!<+&BkFj zg@nOpDneyc+hU9P-;jmOSMN|*H#>^Ez#?;%C3hg_65leSUm;iz)UkW)jX#p)e&S&M z1|a?wDzV5NVnlhRBCd_;F87wp>6c<&nkgvC+!@KGiIqWY4l}=&1w7|r6{oBN8xyzh zG$b#2=RJp_iq6)#t5%yLkKx(0@D=C3w+oiXtSuaQ%I1WIb-eiE$d~!)b@|4XLy!CZ z9p=t=%3ad@Ep+<9003D2KZ5VyP~_n$=;~r&YUg5UZ0KVD&tR1DHy9x)qWtKJp#Kq# zP*8p#W(8JJ_*h_3W}FlvRam?<4Z+-H77^$Lvi+#vmhL9J zJ<1SV45xi;SrO2f=-OB(7#iNA5)x1uNC-yNxUw|!00vcW2PufRm>e~toH;M0Q85MQLWd?3O{i8H+5VkR@l9Dg-ma ze2fZ%>G(u5(k9EHj2L6!;(KZ8%8|*-1V|B#EagbF(rc+5iL_5;Eu)L4Z-V;0HfK4d z*{utLse_rvHZeQ>V5H=f78M3Ntg1BPxFCVD{HbNA6?9*^YIq;B-DJd{Ca2L#)qWP? zvX^NhFmX?CTWw&Ns}lgs;r3i+Bq@y}Ul+U%pzOS0Fcv9~aB(0!>GT0)NO?p=25LjN z2bh>6RhgqD7bQj#k-KOm@JLgMa6>%-ok1WpOe)FS^XOU{c?d5shG(lIn3GiVBxmg`u%-j=)^v&pX1JecJics3&jvPI)mDut52? z3jEA)DM%}BYbxxKrizVYwq?(P&19EXlwD9^-6J+4!}9{ywR9Gk42jjAURAF&EO|~N z)?s>$Da@ikI4|^z0e{r`J8zIs>SpM~Vn^{3fArRu;?+43>lD+^XtUcY1HidJwnR6+ z!;oG2=B6Z_=M%*{z-RaHc(n|1RTKQdNjjV!Pn9lFt^4w|AeN06*j}ZyhqZ^!-=cyGP_ShV1rGxkx8t zB;8`h!S{LD%ot``700d0@Grql(DTt4Awgmi+Yr0@#jbe=2#UkK%rv=OLqF)9D7D1j z!~McAwMYkeaL$~kI~90)5vBhBzWYc3Cj1WI0RS`z000R8-@ET0dA~*r(gSiCJmQMN&4%1D zyVNf0?}sBH8zNbBLn>~(W{d3%@kL_eQ6jEcR{l>C|JK z(R-fA!z|TTRG40|zv}7E@PqCAXP3n`;%|SCQ|ZS%ym$I{`}t3KPL&^l5`3>yah4*6 zifO#{VNz3)?ZL$be;NEaAk9b#{tV?V7 zP|wf5YA*1;s<)9A4~l3BHzG&HH`1xNr#%){4xZ!jq%o=7nN*wMuXlFV{HaiQLJ`5G zBhDi#D(m`Q1pLh@Tq+L;OwuC52RdW7b8}~60WCOK5iYMUad9}7aWBuILb({5=z~YF zt?*Jr5NG+WadM{mDL>GyiByCuR)hd zA=HM?J6l1Xv0Dl+LW@w$OTcEoOda^nFCw*Sy^I@$sSuneMl{4ys)|RY#9&NxW4S)9 zq|%83IpslTLoz~&vTo!Ga@?rj_kw{|k{nv+w&Ku?fyk4Ki4I?);M|5Axm)t+BaE)D zm(`AQ#k^DWrjbuXoJf2{Aj^KT zFb1zMSqxq|vceV+Mf-)$oPflsO$@*A0n0Z!R{&(xh8s}=;t(lIy zv$S8x>m;vQNHuRzoaOo?eiWFe{0;$s`Bc+Osz~}Van${u;g(su`3lJ^TEfo~nERfP z)?aFzpDgnLYiERsKPu|0tq4l2wT)Atr6Qb%m-AUn6HnCue*yWICp7TjW$@sO zm5rm4aTcPQ(rfi7a`xP7cKCFrJD}*&_~xgLyr^-bmsL}y;A5P|al8J3WUoBSjqu%v zxC;mK!g(7r6RRJ852Z~feoC&sD3(6}^5-uLK8o)9{8L_%%rItZK9C){UxB|;G>JbP zsRRtS4-3B*5c+K2kvmgZK8472%l>3cntWUOVHxB|{Ay~aOg5RN;{PJgeVD*H%ac+y!h#wi%o2bF2Ca8IyMyH{>4#{E_8u^@+l-+n=V}Sq?$O z{091@v%Bd*3pk0^2UtiF9Z+(a@wy6 zUdw8J*ze$K#=$48IBi1U%;hmhO>lu!uU;+RS}p&6@rQila7WftH->*A4=5W|Fmtze z)7E}jh@cbmr9iup^i%*(uF%LG&!+Fyl@LFA-}Ca#bxRfDJAiR2dt6644TaYw1Ma79 zt8&DYj31j^5WPNf5P&{)J?WlCe@<3u^78wnd(Ja4^a>{^Tw}W>|Cjt^If|7l^l)^Q zbz|7~CF(k_9~n|h;ysZ+jHzkXf(*O*@5m zLzUmbHp=x!Q|!9NVXyipZ3)^GuIG$k;D)EK!a5=8MFLI_lpf`HPKl=-Ww%z8H_0$j ztJ||IfFG1lE9nmQ0+jPQy zCBdKkjArH@K7jVcMNz);Q(Q^R{d5G?-kk;Uu_IXSyWB)~KGIizZL(^&qF;|1PI7!E zTP`%l)gpX|OFn&)M%txpQ2F!hdA~hX1Cm5)IrdljqzRg!f{mN%G~H1&oqe`5eJCIF zHdD7O;AX-{XEV(a`gBFJ9ews#CVS2y!&>Cm_dm3C8*n3MA*e67(WC?uP@8TXuMroq z{#w$%z@CBIkRM7?}Xib+>hRjy?%G!fiw8! z8(gB+8J~KOU}yO7UGm&1g_MDJ$IXS!`+*b*QW2x)9>K~Y*E&bYMnjl6h!{17_8d!%&9D`a7r&LKZjC<&XOvTRaKJ1 zUY@hl5^R&kZl3lU3njk`3dPzxj$2foOL26r(9zsVF3n_F#v)s5vv3@dgs|lP#eylq62{<-vczqP!RpVBTgI>@O6&sU>W|do17+#OzQ7o5A$ICH z?GqwqnK^n2%LR;$^oZM;)+>$X3s2n}2jZ7CdWIW0lnGK-b#EG01)P@aU`pg}th&J-TrU`tIpb5t((0eu|!u zQz+3ZiOQ^?RxxK4;zs=l8q!-n7X{@jSwK(iqNFiRColuEOg}!7cyZi`iBX4g1pNBj zAPzL?P^Ljhn;1$r8?bc=#n|Ed7wB&oHcw()&*k#SS#h}jO?ZB246EGItsz*;^&tzp zu^YJ0=lwsi`eP_pU8}6JA7MS;9pfD;DsSsLo~ogzMNP70@@;Fm8f0^;>$Z>~}GWRw!W5J3tNX*^2+1f3hz{~rIzJo z6W%J(H!g-eI_J1>0juX$X4Cl6i+3wbc~k146UIX&G22}WE>0ga#WLsn9tY(&29zBvH1$`iWtTe zG2jYl@P!P)eb<5DsR72BdI7-zP&cZNI{7q3e@?N8IKc4DE#UVr->|-ryuJXk^u^>4 z$3wE~=q390;XuOQP~TNoDR?#|NSPJ%sTMInA6*rJ%go|=YjGe!B>z6u$IhgQSwoV* zjy3F2#I>uK{42{&IqP59)Y(1*Z>>#W8rCf4_eVsH)`v!P#^;BgzKDR`ARGEZzkNX+ zJUQu=*-ol=Xqqt5=`=pA@BIn@6a9G8C{c&`i^(i+BxQO9?YZ3iu%$$da&Kb?2kCCo zo7t$UpSFWqmydXf@l3bVJ=%K?SSw)|?srhJ-1ZdFu*5QhL$~-IQS!K1s@XzAtv6*Y zl8@(5BlWYLt1yAWy?rMD&bwze8bC3-GfNH=p zynNFCdxyX?K&G(ZZ)afguQ2|r;XoV^=^(;Cku#qYn4Lus`UeKt6rAlFo_rU`|Rq z&G?~iWMBio<78of-2X(ZYHx~=U0Vz4btyXkctMKdc9UM!vYr~B-(>)(Hc|D zMzkN4!PBg%tZoh+=Gba!0++d193gbMk2&krfDgcbx0jI92cq?FFESVg0D$>F+bil} zY~$)|>1HZsX=5sAZ2WgPB5P=8X#TI+NQ(M~GqyVB53c6IdX=k>Wu@A0Svf5#?uHaF zsYn|koIi3$(%GZ2+G+7Fv^lHTb#5b8sAHSTnL^qWZLM<(1|9|QFw9pnRU{svj}_Al zL)b9>fN{QiA($8peNEJyy`(a{&uh-T4_kdZFIVsKKVM(?05}76EEz?#W za^fiZOAd14IJ4zLX-n7Lq0qlQ^lW8Cvz4UKkV9~P}>sq0?xD3vg+$4vLm~C(+ zM{-3Z#qnZ09bJ>}j?6ry^h+@PfaD7*jZxBEY4)UG&daWb??6)TP+|3#Z&?GL?1i+280CFsE|vIXQbm| zM}Pk!U`U5NsNbyKzkrul-DzwB{X?n3E6?TUHr{M&+R*2%yOiXdW-_2Yd6?38M9Vy^ z*lE%gA{wwoSR~vN0=no}tP2Ul5Gk5M(Xq`$nw#ndFk`tcpd5A=Idue`XZ!FS>Q zG^0w#>P4pPG+*NC9gLP4x2m=cKP}YuS!l^?sHSFftZy{4CoQrb_ z^20(NnG`wAhMI=eq)SsIE~&Gp9Ne0nD4%Xiu|0Fj1UFk?6avDqjdXz{O1nKao*46y zT8~iA%Exu=G#{x=KD;_C&M+Zx4+n`sHT>^>=-1YM;H<72k>$py1?F3#T1*ef9mLZw z5naLQr?n7K;2l+{_uIw*_1nsTn~I|kkCgrn;|G~##hM;9l7Jy$yJfmk+&}W@JeKcF zx@@Woiz8qdi|D%aH3XTx5*wDlbs?dC1_nrFpm^QbG@wM=i2?Zg;$VK!c^Dp8<}BTI zyRhAq@#%2pGV49*Y5_mV4+OICP|%I(dQ7x=6Ob}>EjnB_-_18*xrY?b%-yEDT(wrO z9RY2QT0`_OpGfMObKHV;QLVnrK%mc?$WAdIT`kJQT^n%GuzE7|9@k3ci5fYOh(287 zuIbg!GB3xLg$YN=n)^pHGB0jH+_iIiC=nUcD;G6LuJsjn2VI1cyZx=a?ShCsF==QK z;q~*m&}L<-cb+mDDXzvvrRsybcgQ;Vg21P(uLv5I+eGc7o7tc6`;OA9{soHFOz zT~2?>Ts}gprIX$wRBb4yE>ot<8+*Bv`qbSDv*VtRi|cyWS>)Fjs>fkNOH-+PX&4(~ z&)T8Zam2L6puQl?;5zg9h<}k4#|yH9czHw;1jw-pwBM*O2hUR6yvHATrI%^mvs9q_ z&ccT0>f#eDG<^WG^q@oVqlJrhxH)dcq2cty@l3~|5#UDdExyXUmLQ}f4#;6fI{f^t zDCsgIJ~0`af%YR%Ma5VQq-p21k`vaBu6WE?66+5=XUd%Ay%D$irN>5LhluRWt7 zov-=f>QbMk*G##&DTQyou$s7UqjjW@k6=!I@!k+S{pP8R(2=e@io;N8E`EOB;OGoI zw6Q+{X1_I{OO0HPpBz!X!@`5YQ2)t{+!?M_iH25X(d~-Zx~cXnS9z>u?+If|iNJbx zyFU2d1!ITX64D|lE0Z{dLRqL1Ajj=CCMfC4lD3&mYR_R_VZ>_7_~|<^o*%_&jevU+ zQ4|qzci=0}Jydw|LXLCrOl1_P6Xf@c0$ieK2^7@A9UbF{@V_0p%lqW|L?5k>bVM8|p5v&2g;~r>B8uo<4N+`B zH{J)h;SYiIVx@#jI&p-v3dwL5QNV1oxPr8J%ooezTnLW>i*3Isb49%5i!&ac_dEXv zvXmVUck^QHmyrF8>CGXijC_R-y(Qr{3Zt~EmW)-nC!tiH`wlw5D*W7Pip;T?&j%kX z6DkZX4&}iw>hE(boLyjOoupf6JpvBG8}jIh!!VhnD0>}KSMMo{1#uU6kiFcA04~|7 zVO8eI&x1`g4CZ<2cYUI(n#wz2MtVFHx47yE5eL~8bot~>EHbevSt}LLMQX?odD{Ux zJMnam{d)W4da{l7&y-JrgiU~qY3$~}_F#G7|MxT)e;G{U`In&?`j<5D->}cb{}{T(4DF0BOk-=1195KB-E*o@c?`>y#4=dMtYtSY=&L{!TAjFVcq0y@AH`vH! z$41+u!Ld&}F^COPgL(EE{0X7LY&%D7-(?!kjFF7=qw<;`V{nwWBq<)1QiGJgUc^Vz ztMUlq1bZqKn17|6x6iAHbWc~l1HcmAxr%$Puv!znW)!JiukwIrqQ00|H$Z)OmGG@= zv%A8*4cq}(?qn4rN6o`$Y))(MyXr8R<2S^J+v(wmFmtac!%VOfN?&(8Nr!T@kV`N; z*Q33V3t`^rN&aBiHet)18wy{*wi1=W!B%B-Q6}SCrUl$~Hl{@!95ydml@FK8P=u4s z4e*7gV2s=YxEvskw2Ju!2%{8h01rx-3`NCPc(O zH&J0VH5etNB2KY6k4R@2Wvl^Ck$MoR3=)|SEclT2ccJ!RI9Nuter7u9@;sWf-%um;GfI!=eEIQ2l2p_YWUd{|6EG ze{yO6;lMc>;2tPrsNdi@&1K6(1;|$xe8vLgiouj%QD%gYk`4p{Ktv9|j+!OF-P?@p z;}SV|oIK)iwlBs+`ROXkhd&NK zzo__r!B>tOXpBJMDcv!Mq54P+n4(@dijL^EpO1wdg~q+!DT3lB<>9AANSe!T1XgC=J^)IP0XEZ()_vpu!!3HQyJhwh?r`Ae%Yr~b% zO*NY9t9#qWa@GCPYOF9aron7thfWT`eujS4`t2uG6)~JRTI;f(ZuoRQwjZjp5Pg34 z)rp$)Kr?R+KdJ;IO;pM{$6|2y=k_siqvp%)2||cHTe|b5Ht8&A{wazGNca zX$Ol?H)E_R@SDi~4{d-|8nGFhZPW;Cts1;08TwUvLLv&_2$O6Vt=M)X;g%HUr$&06 zISZb(6)Q3%?;3r~*3~USIg=HcJhFtHhIV(siOwV&QkQe#J%H9&E21!C*d@ln3E@J* zVqRO^<)V^ky-R|%{(9`l-(JXq9J)1r$`uQ8a}$vr9E^nNiI*thK8=&UZ0dsFN_eSl z(q~lnD?EymWLsNa3|1{CRPW60>DSkY9YQ;$4o3W7Ms&@&lv9eH!tk~N&dhqX&>K@} zi1g~GqglxkZ5pEFkllJ)Ta1I^c&Bt6#r(QLQ02yHTaJB~- zCcE=5tmi`UA>@P=1LBfBiqk)HB4t8D?02;9eXj~kVPwv?m{5&!&TFYhu>3=_ zsGmYZ^mo*-j69-42y&Jj0cBLLEulNRZ9vXE)8~mt9C#;tZs;=#M=1*hebkS;7(aGf zcs7zH(I8Eui9UU4L--))yy`&d&$In&VA2?DAEss4LAPCLd>-$i?lpXvn!gu^JJ$(DoUlc6wE98VLZ*z`QGQov5l4Fm_h?V-;mHLYDVOwKz7>e4+%AzeO>P6v}ndPW| zM>m#6Tnp7K?0mbK=>gV}=@k*0Mr_PVAgGMu$j+pWxzq4MAa&jpCDU&-5eH27Iz>m^ zax1?*HhG%pJ((tkR(V(O(L%7v7L%!_X->IjS3H5kuXQT2!ow(;%FDE>16&3r){!ex zhf==oJ!}YU89C9@mfDq!P3S4yx$aGB?rbtVH?sHpg?J5C->!_FHM%Hl3#D4eplxzQ zRA+<@LD%LKSkTk2NyWCg7u=$%F#;SIL44~S_OGR}JqX}X+=bc@swpiClB`Zbz|f!4 z7Ysah7OkR8liXfI`}IIwtEoL}(URrGe;IM8%{>b1SsqXh)~w}P>yiFRaE>}rEnNkT z!HXZUtxUp1NmFm)Dm@-{FI^aRQqpSkz}ZSyKR%Y}YHNzBk)ZIp} zMtS=aMvkgWKm9&oTcU0?S|L~CDqA+sHpOxwnswF-fEG)cXCzUR?ps@tZa$=O)=L+5 zf%m58cq8g_o}3?Bhh+c!w4(7AjxwQ3>WnVi<{{38g7yFboo>q|+7qs<$8CPXUFAN< zG&}BHbbyQ5n|qqSr?U~GY{@GJ{(Jny{bMaOG{|IkUj7tj^9pa9|FB_<+KHLxSxR;@ zHpS$4V)PP+tx}22fWx(Ku9y+}Ap;VZqD0AZW4gCDTPCG=zgJmF{|x;(rvdM|2|9a}cex6xrMkERnkE;}jvU-kmzd%_J50$M`lIPCKf+^*zL=@LW`1SaEc%=m zQ+lT06Gw+wVwvQ9fZ~#qd430v2HndFsBa9WjD0P}K(rZYdAt^5WQIvb%D^Q|pkVE^ zte$&#~zmULFACGfS#g=2OLOnIf2Of-k!(BIHjs77nr!5Q1*I9 z1%?=~#Oss!rV~?-6Gm~BWJiA4mJ5TY&iPm_$)H1_rTltuU1F3I(qTQ^U$S>%$l z)Wx1}R?ij0idp@8w-p!Oz{&*W;v*IA;JFHA9%nUvVDy7Q8woheC#|8QuDZb-L_5@R zOqHwrh|mVL9b=+$nJxM`3eE{O$sCt$UK^2@L$R(r^-_+z?lOo+me-VW=Zw z-Bn>$4ovfWd%SPY`ab-u9{INc*k2h+yH%toDHIyqQ zO68=u`N}RIIs7lsn1D){)~%>ByF<>i@qFb<-axvu(Z+6t7v<^z&gm9McRB~BIaDn$ z#xSGT!rzgad8o>~kyj#h1?7g96tOcCJniQ+*#=b7wPio>|6a1Z?_(TS{)KrPe}(8j z!#&A=k(&Pj^F;r)CI=Z{LVu>uj!_W1q4b`N1}E(i%;BWjbEcnD=mv$FL$l?zS6bW!{$7j1GR5ocn94P2u{ z70tAAcpqtQo<@cXw~@i-@6B23;317|l~S>CB?hR5qJ%J3EFgyBdJd^fHZu7AzHF(BQ!tyAz^L0`X z23S4Fe{2X$W0$zu9gm%rg~A>ijaE#GlYlrF9$ds^QtaszE#4M(OLVP2O-;XdT(XIC zatwzF*)1c+t~c{L=fMG8Z=k5lv>U0;C{caN1NItnuSMp)6G3mbahu>E#sj&oy94KC zpH}8oEw{G@N3pvHhp{^-YaZeH;K+T_1AUv;IKD<=mv^&Ueegrb!yf`4VlRl$M?wsl zZyFol(2|_QM`e_2lYSABpKR{{NlxlDSYQNkS;J66aT#MSiTx~;tUmvs-b*CrR4w=f z8+0;*th6kfZ3|5!Icx3RV11sp=?`0Jy3Fs0N4GZQMN=8HmT6%x9@{Dza)k}UwL6JT zHRDh;%!XwXr6yuuy`4;Xsn0zlR$k%r%9abS1;_v?`HX_hI|+EibVnlyE@3aL5vhQq zlIG?tN^w@0(v9M*&L+{_+RQZw=o|&BRPGB>e5=ys7H`nc8nx)|-g;s7mRc7hg{GJC zAe^vCIJhajmm7C6g! zL&!WAQ~5d_5)00?w_*|*H>3$loHrvFbitw#WvLB!JASO?#5Ig5$Ys10n>e4|3d;tS zELJ0|R4n3Az(Fl3-r^QiV_C;)lQ1_CW{5bKS15U|E9?ZgLec@%kXr84>5jV2a5v=w z?pB1GPdxD$IQL4)G||B_lI+A=08MUFFR4MxfGOu07vfIm+j=z9tp~5i_6jb`tR>qV z$#`=BQ*jpCjm$F0+F)L%xRlnS%#&gro6PiRfu^l!EVan|r3y}AHJQOORGx4~ z&<)3=K-tx518DZyp%|!EqpU!+X3Et7n2AaC5(AtrkW>_57i}$eqs$rupubg0a1+WO zGHZKLN2L0D;ab%{_S1Plm|hx8R?O14*w*f&2&bB050n!R2by zw!@XOQx$SqZ5I<(Qu$V6g>o#A!JVwErWv#(Pjx=KeS0@hxr4?13zj#oWwPS(7Ro|v z>Mp@Kmxo79q|}!5qtX2-O@U&&@6s~!I&)1WQIl?lTnh6UdKT_1R640S4~f=_xoN3- zI+O)$R@RjV$F=>Ti7BlnG1-cFKCC(t|Qjm{SalS~V-tX#+2ekRhwmN zZr`8{QF6y~Z!D|{=1*2D-JUa<(1Z=;!Ei!KiRNH?o{p5o3crFF=_pX9O-YyJchr$~ zRC`+G+8kx~fD2k*ZIiiIGR<8r&M@3H?%JVOfE>)})7ScOd&?OjgAGT@WVNSCZ8N(p zuQG~76GE3%(%h1*vUXg$vH{ua0b`sQ4f0*y=u~lgyb^!#CcPJa2mkSEHGLsnO^kb$ zru5_l#nu=Y{rSMWiYx?nO{8I!gH+?wEj~UM?IrG}E|bRIBUM>UlY<`T1EHpRr36vv zBi&dG8oxS|J$!zoaq{+JpJy+O^W(nt*|#g32bd&K^w-t>!Vu9N!k9eA8r!Xc{utY> zg9aZ(D2E0gL#W0MdjwES-7~Wa8iubPrd?8-$C4BP?*wok&O8+ykOx{P=Izx+G~hM8 z*9?BYz!T8~dzcZr#ux8kS7u7r@A#DogBH8km8Ry4slyie^n|GrTbO|cLhpqgMdsjX zJ_LdmM#I&4LqqsOUIXK8gW;V0B(7^$y#h3h>J0k^WJfAMeYek%Y-Dcb_+0zPJez!GM zAmJ1u;*rK=FNM0Nf}Y!!P9c4)HIkMnq^b;JFd!S3?_Qi2G#LIQ)TF|iHl~WKK6JmK zbv7rPE6VkYr_%_BT}CK8h=?%pk@3cz(UrZ{@h40%XgThP*-Oeo`T0eq9 zA8BnWZKzCy5e&&_GEsU4*;_k}(8l_&al5K-V*BFM=O~;MgRkYsOs%9eOY6s6AtE*<7GQAR2ulC3RAJrG_P1iQK5Z~&B z&f8X<>yJV6)oDGIlS$Y*D^Rj(cszTy5c81a5IwBr`BtnC6_e`ArI8CaTX_%rx7;cn zR-0?J_LFg*?(#n~G8cXut(1nVF0Oka$A$1FGcERU<^ggx;p@CZc?3UB41RY+wLS`LWFNSs~YP zuw1@DNN3lTd|jDL7gjBsd9}wIw}4xT2+8dBQzI00m<@?c2L%>}QLfK5%r!a-iII`p zX@`VEUH)uj^$;7jVUYdADQ2k*!1O3WdfgF?OMtUXNpQ1}QINamBTKDuv19^{$`8A1 zeq%q*O0mi@(%sZU>Xdb0Ru96CFqk9-L3pzLVsMQ`Xpa~N6CR{9Rm2)A|CI21L(%GW zh&)Y$BNHa=FD+=mBw3{qTgw)j0b!Eahs!rZnpu)z!!E$*eXE~##yaXz`KE5(nQM`s zD!$vW9XH)iMxu9R>r$VlLk9oIR%HxpUiW=BK@4U)|1WNQ=mz9a z^!KkO=>GaJ!GBXm{KJj^;kh-MkUlEQ%lza`-G&}C5y1>La1sR6hT=d*NeCnuK%_LV zOXt$}iP6(YJKc9j-Fxq~*ItVUqljQ8?oaysB-EYtFQp9oxZ|5m0^Hq(qV!S+hq#g( z?|i*H2MIr^Kxgz+3vIljQ*Feejy6S4v~jKEPTF~Qhq!(ms5>NGtRgO5vfPPc4Z^AM zTj!`5xEreIN)vaNxa|q6qWdg>+T`Ol0Uz)ckXBXEGvPNEL3R8hB3=C5`@=SYgAju1 z!)UBr{2~=~xa{b8>x2@C7weRAEuatC)3pkRhT#pMPTpSbA|tan%U7NGMvzmF?c!V8 z=pEWxbdXbTAGtWTyI?Fml%lEr-^AE}w#l(<7OIw;ctw}imYax&vR4UYNJZK6P7ZOd zP87XfhnUHxCUHhM@b*NbTi#(-8|wcv%3BGNs#zRCVV(W?1Qj6^PPQa<{yaBwZ`+<`w|;rqUY_C z&AeyKwwf*q#OW-F()lir=T^<^wjK65Lif$puuU5+tk$;e_EJ;Lu+pH>=-8=PDhkBg z8cWt%@$Sc#C6F$Vd+0507;{OOyT7Hs%nKS88q-W!$f~9*WGBpHGgNp}=C*7!RiZ5s zn1L_DbKF@B8kwhDiLKRB@lsXVVLK|ph=w%_`#owlf@s@V(pa`GY$8h%;-#h@TsO|Y8V=n@*!Rog7<7Cid%apR|x zOjhHCyfbIt%+*PCveTEcuiDi%Wx;O;+K=W?OFUV%)%~6;gl?<0%)?snDDqIvkHF{ zyI02)+lI9ov42^hL>ZRrh*HhjF9B$A@=H94iaBESBF=eC_KT$8A@uB^6$~o?3Wm5t1OIaqF^~><2?4e3c&)@wKn9bD? zoeCs;H>b8DL^F&>Xw-xjZEUFFTv>JD^O#1E#)CMBaG4DX9bD(Wtc8Rzq}9soQ8`jf zeSnHOL}<+WVSKp4kkq&?SbETjq6yr@4%SAqOG=9E(3YeLG9dtV+8vmzq+6PFPk{L; z(&d++iu=^F%b+ea$i2UeTC{R*0Isk;vFK!no<;L+(`y`3&H-~VTdKROkdyowo1iqR zbVW(3`+(PQ2>TKY>N!jGmGo7oeoB8O|P_!Ic@ zZ^;3dnuXo;WJ?S+)%P>{Hcg!Jz#2SI(s&dY4QAy_vRlmOh)QHvs_7c&zkJCmJGVvV zX;Mtb>QE+xp`KyciG$Cn*0?AK%-a|=o!+7x&&yzHQOS>8=B*R=niSnta^Pxp1`=md z#;$pS$4WCT?mbiCYU?FcHGZ#)kHVJTTBt^%XE(Q};aaO=Zik0UgLcc0I(tUpt(>|& zcxB_|fxCF7>&~5eJ=Dpn&5Aj{A^cV^^}(7w#p;HG&Q)EaN~~EqrE1qKrMAc&WXIE;>@<&)5;gD2?={Xf@Mvn@OJKw=8Mgn z!JUFMwD+s==JpjhroT&d{$kQAy%+d`a*XxDEVxy3`NHzmITrE`o!;5ClXNPb4t*8P zzAivdr{j_v!=9!^?T3y?gzmqDWX6mkzhIzJ-3S{T5bcCFMr&RPDryMcdwbBuZbsgN zGrp@^i?rcfN7v0NKGzDPGE#4yszxu=I_`MI%Z|10nFjU-UjQXXA?k8Pk|OE<(?ae) zE%vG#eZAlj*E7_3dx#Zz4kMLj>H^;}33UAankJiDy5ZvEhrjr`!9eMD8COp}U*hP+ zF}KIYx@pkccIgyxFm#LNw~G&`;o&5)2`5aogs`1~7cMZQ7zj!%L4E`2yzlQN6REX20&O<9 zKV6fyr)TScJPPzNTC2gL+0x#=u>(({{D7j)c-%tvqls3#Y?Z1m zV5WUE)zdJ{$p>yX;^P!UcXP?UD~YM;IRa#Rs5~l+*$&nO(;Ers`G=0D!twR(0GF@c zHl9E5DQI}Oz74n zfKP>&$q0($T4y$6w(p=ERAFh+>n%iaeRA%!T%<^+pg?M)@ucY<&59$x9M#n+V&>}=nO9wCV{O~lg&v#+jcUj(tQ z`0u1YH)-`U$15a{pBkGyPL0THv1P|4e@pf@3IBZS4dVJPo#H>pWq%Lr0YS-SeWash z8R7=jb28KPMI|_lo#GEO|5B?N_e``H*23{~a!AmUJ+fb4HX-%QI@lSEUxKlGV7z7Q zSKw@-TR>@1RL%w{x}dW#k1NgW+q4yt2Xf1J62Bx*O^WG8OJ|FqI4&@d3_o8Id@*)4 zYrk=>@!wv~mh7YWv*bZhxqSmFh2Xq)o=m;%n$I?GSz49l1$xRpPu_^N(vZ>*>Z<04 z2+rP70oM=NDysd!@fQdM2OcyT?3T^Eb@lIC-UG=Bw{BjQ&P`KCv$AcJ;?`vdZ4){d z&gkoUK{$!$$K`3*O-jyM1~p-7T*qb)Ys>Myt^;#1&a%O@x8A+E>! zY8=eD`ZG)LVagDLBeHg>=atOG?Kr%h4B%E6m@J^C+U|y)XX@f z8oyJDW|9g=<#f<{JRr{y#~euMnv)`7j=%cHWLc}ngjq~7k**6%4u>Px&W%4D94(r* z+akunK}O0DC2A%Xo9jyF;DobX?!1I(7%}@7F>i%&nk*LMO)bMGg2N+1iqtg+r(70q zF5{Msgsm5GS7DT`kBsjMvOrkx&|EU!{{~gL4d2MWrAT=KBQ-^zQCUq{5PD1orxlIL zq;CvlWx#f1NWvh`hg011I%?T_s!e38l*lWVt|~z-PO4~~1g)SrJ|>*tXh=QfXT)%( z+ex+inPvD&O4Ur;JGz>$sUOnWdpSLcm1X%aQDw4{dB!cnj`^muI$CJ2%p&-kULVCE z>$eMR36kN$wCPR+OFDM3-U(VOrp9k3)lI&YVFqd;Kpz~K)@Fa&FRw}L(SoD z9B4a+hQzZT-BnVltst&=kq6Y(f^S4hIGNKYBgMxGJ^;2yrO}P3;r)(-I-CZ)26Y6? z&rzHI_1GCvGkgy-t1E;r^3Le30|%$ebDRu2+gdLG)r=A~Qz`}~&L@aGJ{}vVs_GE* zVUjFnzHiXfKQbpv&bR&}l2bzIjAooB)=-XNcYmrGmBh(&iu@o!^hn0^#}m2yZZUK8 zufVm7Gq0y`Mj;9b>`c?&PZkU0j4>IL=UL&-Lp3j&47B5pAW4JceG{!XCA)kT<%2nqCxj<)uy6XR_uws~>_MEKPOpAQ!H zkn>FKh)<9DwwS*|Y(q?$^N!6(51O0 z^JM~Ax{AI1Oj$fs-S5d4T7Z_i1?{%0SsIuQ&r8#(JA=2iLcTN+?>wOL532%&dMYkT z*T5xepC+V6zxhS@vNbMoi|i)=rpli@R9~P!39tWbSSb904ekv7D#quKbgFEMTb48P zuq(VJ+&L8aWU(_FCD$3^uD!YM%O^K(dvy~Wm2hUuh6bD|#(I39Xt>N1Y{ZqXL`Fg6 zKQ?T2htHN!(Bx;tV2bfTtIj7e)liN-29s1kew>v(D^@)#v;}C4-G=7x#;-dM4yRWm zyY`cS21ulzMK{PoaQ6xChEZ}o_#}X-o}<&0)$1#3we?+QeLt;aVCjeA)hn!}UaKt< zat1fHEx13y-rXNMvpUUmCVzocPmN~-Y4(YJvQ#db)4|%B!rBsgAe+*yor~}FrNH08 z3V!97S}D7d$zbSD{$z;@IYMxM6aHdypIuS*pr_U6;#Y!_?0i|&yU*@16l z*dcMqDQgfNBf}?quiu4e>H)yTVfsp#f+Du0@=Kc41QockXkCkvu>FBd6Q+@FL!(Yx z2`YuX#eMEiLEDhp+9uFqME_E^faV&~9qjBHJkIp~%$x^bN=N)K@kvSVEMdDuzA0sn z88CBG?`RX1@#hQNd`o^V{37)!w|nA)QfiYBE^m=yQKv-fQF+UCMcuEe1d4BH7$?>b zJl-r9@0^Ie=)guO1vOd=i$_4sz>y3x^R7n4ED!5oXL3@5**h(xr%Hv)_gILarO46q+MaDOF%ChaymKoI6JU5Pg;7#2n9-18|S1;AK+ zgsn6;k6-%!QD>D?cFy}8F;r@z8H9xN1jsOBw2vQONVqBVEbkiNUqgw~*!^##ht>w0 zUOykwH=$LwX2j&nLy=@{hr)2O&-wm-NyjW7n~Zs9UlH;P7iP3 zI}S(r0YFVYacnKH(+{*)Tbw)@;6>%=&Th=+Z6NHo_tR|JCI8TJiXv2N7ei7M^Q+RM z?9o`meH$5Yi;@9XaNR#jIK^&{N|DYNNbtdb)XW1Lv2k{E>;?F`#Pq|&_;gm~&~Zc9 zf+6ZE%{x4|{YdtE?a^gKyzr}dA>OxQv+pq|@IXL%WS0CiX!V zm$fCePA%lU{%pTKD7|5NJHeXg=I0jL@$tOF@K*MI$)f?om)D63K*M|r`gb9edD1~Y zc|w7N)Y%do7=0{RC|AziW7#am$)9jciRJ?IWl9PE{G3U+$%FcyKs_0Cgq`=K3@ttV z9g;M!3z~f_?P%y3-ph%vBMeS@p7P&Ea8M@97+%XEj*(1E6vHj==d zjsoviB>j^$_^OI_DEPvFkVo(BGRo%cJeD){6Uckei=~1}>sp299|IRjhXe)%?uP0I zF5+>?0#Ye}T^Y$u_rc4=lPcq4K^D(TZG-w30-YiEM=dcK+4#o*>lJ8&JLi+3UcpZk z!^?95S^C0ja^jwP`|{<+3cBVog$(mRdQmadS+Vh~z zS@|P}=|z3P6uS+&@QsMp0no9Od&27O&14zHXGAOEy zh~OKpymK5C%;LLb467@KgIiVwYbYd6wFxI{0-~MOGfTq$nBTB!{SrWmL9Hs}C&l&l#m?s*{tA?BHS4mVKHAVMqm63H<|c5n0~k)-kbg zXidai&9ZUy0~WFYYKT;oe~rytRk?)r8bptITsWj(@HLI;@=v5|XUnSls7$uaxFRL+ zRVMGuL3w}NbV1`^=Pw*0?>bm8+xfeY(1PikW*PB>>Tq(FR`91N0c2&>lL2sZo5=VD zQY{>7dh_TX98L2)n{2OV=T10~*YzX27i2Q7W86M4$?gZIXZaBq#sA*{PH8){|GUi;oM>e?ua7eF4WFuFYZSG| zze?srg|5Ti8Og{O zeFxuw9!U+zhyk?@w zjsA6(oKD=Ka;A>Ca)oPORxK+kxH#O@zhC!!XS4@=swnuMk>t+JmLmFiE^1aX3f<)D@`%K0FGK^gg1a1j>zi z2KhV>sjU7AX3F$SEqrXSC}fRx64GDoc%!u2Yag68Lw@w9v;xOONf@o)Lc|Uh3<21ctTYu-mFZuHk*+R{GjXHIGq3p)tFtQp%TYqD=j1&y)>@zxoxUJ!G@ zgI0XKmP6MNzw>nRxK$-Gbzs}dyfFzt>#5;f6oR27ql!%+{tr+(`(>%51|k`ML} zY4eE)Lxq|JMas(;JibNQds1bUB&r}ydMQXBY4x(^&fY_&LlQC)3hylc$~8&~|06-D z#T+%66rYbHX%^KuqJED_wuGB+=h`nWA!>1n0)3wZrBG3%`b^Ozv6__dNa@%V14|!D zQ?o$z5u0^8`giv%qE!BzZ!3j;BlDlJDk)h@9{nSQeEk!z9RGW) z${RSF3phEM*ce*>Xdp}585vj$|40=&S{S-GTiE?Op*vY&Lvr9}BO$XWy80IF+6@%n z5*2ueT_g@ofP#u5pxb7n*fv^Xtt7&?SRc{*2Ka-*!BuOpf}neHGCiHy$@Ka1^Dint z;DkmIL$-e)rj4o2WQV%Gy;Xg(_Bh#qeOsTM2f@KEe~4kJ8kNLQ+;(!j^bgJMcNhvklP5Z6I+9Fq@c&D~8Fb-4rmDT!MB5QC{Dsb;BharP*O;SF4& zc$wj-7Oep7#$WZN!1nznc@Vb<_Dn%ga-O#J(l=OGB`dy=Sy&$(5-n3zzu%d7E#^8`T@}V+5B;PP8J14#4cCPw-SQTdGa2gWL0*zKM z#DfSXs_iWOMt)0*+Y>Lkd=LlyoHjublNLefhKBv@JoC>P7N1_#> zv=mLWe96%EY;!ZGSQDbZWb#;tzqAGgx~uk+-$+2_8U`!ypbwXl z^2E-FkM1?lY@yt8=J3%QK+xaZ6ok=-y%=KXCD^0r!5vUneW>95PzCkOPO*t}p$;-> ze5j-BLT_;)cZQzR2CEsm@rU7GZfFtdp*a|g4wDr%8?2QkIGasRfDWT-Dvy*U{?IHT z*}wGnzdlSptl#ZF^sf)KT|BJs&kLG91^A6ls{CzFprZ6-Y!V0Xysh%9p%iMd7HLsS zN+^Un$tDV)T@i!v?3o0Fsx2qI(AX_$dDkBzQ@fRM%n zRXk6hb9Py#JXUs+7)w@eo;g%QQ95Yq!K_d=z{0dGS+pToEI6=Bo8+{k$7&Z zo4>PH(`ce8E-Ps&uv`NQ;U$%t;w~|@E3WVOCi~R4oj5wP?%<*1C%}Jq%a^q~T7u>K zML5AKfQDv6>PuT`{SrKHRAF+^&edg6+5R_#H?Lz3iGoWo#PCEd0DS;)2U({{X#zU^ zw_xv{4x7|t!S)>44J;KfA|DC?;uQ($l+5Vp7oeqf7{GBF9356nx|&B~gs+@N^gSdd zvb*>&W)|u#F{Z_b`f#GVtQ`pYv3#||N{xj1NgB<#=Odt6{eB%#9RLt5v zIi|0u70`#ai}9fJjKv7dE!9ZrOIX!3{$z_K5FBd-Kp-&e4(J$LD-)NMTp^_pB`RT; zftVVlK2g@+1Ahv2$D){@Y#cL#dUj9*&%#6 zd2m9{1NYp>)6=oAvqdCn5#cx{AJ%S8skUgMglu2*IAtd+z1>B&`MuEAS(D(<6X#Lj z?f4CFx$)M&$=7*>9v1ER4b6!SIz-m0e{o0BfkySREchp?WdVPpQCh!q$t>?rL!&Jg zd#heM;&~A}VEm8Dvy&P|J*eAV&w!&Nx6HFV&B8jJFVTmgLaswn!cx$&%JbTsloz!3 zMEz1d`k==`Ueub_JAy_&`!ogbwx27^ZXgFNAbx=g_I~5nO^r)}&myw~+yY*cJl4$I znNJ32M&K=0(2Dj_>@39`3=FX!v3nZHno_@q^!y}%(yw0PqOo=);6Y@&ylVe>nMOZ~ zd>j#QQSBn3oaWd;qy$&5(5H$Ayi)0haAYO6TH>FR?rhqHmNOO+(})NB zLI@B@v0)eq!ug`>G<@htRlp3n!EpU|n+G+AvXFrWSUsLMBfL*ZB`CRsIVHNTR&b?K zxBgsN0BjfB>UVcJ|x%=-zb%OV7lmZc& zxiupadZVF7)6QuhoY;;FK2b*qL0J-Rn-8!X4ZY$-ZSUXV5DFd7`T41c(#lAeLMoeT z4%g655v@7AqT!i@)Edt5JMbN(=Q-6{=L4iG8RA%}w;&pKmtWvI4?G9pVRp|RTw`g0 zD5c12B&A2&P6Ng~8WM2eIW=wxd?r7A*N+&!Be7PX3s|7~z=APxm=A?5 zt>xB4WG|*Td@VX{Rs)PV0|yK`oI3^xn(4c_j&vgxk_Y3o(-`_5o`V zRTghg6%l@(qodXN;dB#+OKJEEvhfcnc#BeO2|E(5df-!fKDZ!%9!^BJ_4)9P+9Dq5 zK1=(v?KmIp34r?z{NEWnLB3Px{XYwy-akun4F7xTRr2^zeYW{gcK9)>aJDdU5;w5@ zak=<+-PLH-|04pelTb%ULpuuuJC7DgyT@D|p{!V!0v3KpDnRjANN12q6SUR3mb9<- z>2r~IApQGhstZ!3*?5V z8#)hJ0TdZg0M-BK#nGFP>$i=qk82DO z7h;Ft!D5E15OgW)&%lej*?^1~2=*Z5$2VX>V{x8SC+{i10BbtUk9@I#Vi&hX)q
Q!LwySI{Bnv%Sm)yh{^sSVJ8&h_D-BJ_YZe5eCaAWU9b$O2c z$T|{vWVRtOL!xC0DTc(Qbe`ItNtt5hr<)VijD0{U;T#bUEp381_y`%ZIav?kuYG{iyYdEBPW=*xNSc;Rlt6~F4M`5G+VtOjc z*0qGzCb@gME5udTjJA-9O<&TWd~}ysBd(eVT1-H82-doyH9RST)|+Pb{o*;$j9Tjs zhU!IlsPsj8=(x3bAKJTopW3^6AKROHR^7wZ185wJGVhA~hEc|LP;k7NEz-@4p5o}F z`AD6naG3(n=NF9HTH81=F+Q|JOz$7wm9I<+#BSmB@o_cLt2GkW9|?7mM;r!JZp89l zbo!Hp8=n!XH1{GwaDU+k)pGp`C|cXkCU5%vcH)+v@0eK>%7gWxmuMu9YLlChA|_D@ zi#5zovN_!a-0?~pUV-Rj*1P)KwdU-LguR>YM&*Nen+ln8Q$?WFCJg%DY%K}2!!1FE zDv-A%Cbwo^p(lzac&_TZ-l#9kq`mhLcY3h9ZTUVCM(Ad&=EriQY5{jJv<5K&g|*Lk zgV%ILnf1%8V2B0E&;Sp4sYbYOvvMebLwYwzkRQ#F8GpTQq#uv=J`uaSJ34OWITeSGo6+-8Xw znCk*n{kdDEi)Hi&u^)~cs@iyCkFWB2SWZU|Uc%^43ZIZQ-vWNExCCtDWjqHs;;tWf$v{}0{p0Rvxkq``)*>+Akq%|Na zA`@~-Vfe|+(AIlqru+7Ceh4nsVmO9p9jc8}HX^W&ViBDXT+uXbT#R#idPn&L>+#b6 zflC-4C5-X;kUnR~L>PSLh*gvL68}RBsu#2l`s_9KjUWRhiqF`j)`y`2`YU(>3bdBj z?>iyjEhe-~$^I5!nn%B6Wh+I`FvLNvauve~eX<+Ipl&04 zT}};W&1a3%W?dJ2=N#0t?e+aK+%t}5q%jSLvp3jZ%?&F}nOOWr>+{GFIa%wO_2`et z=JzoRR~}iKuuR+azPI8;Gf9)z3kyA4EIOSl!sRR$DlW}0>&?GbgPojmjmnln;cTqCt=ADbE zZ8GAnoM+S1(5$i8^O4t`ue;vO4i}z0wz-QEIVe5_u03;}-!G1NyY8;h^}y;tzY}i5 zqQr#Ur3Fy8sSa$Q0ys+f`!`+>9WbvU_I`Sj;$4{S>O3?#inLHCrtLy~!s#WXV=oVP zeE93*Nc`PBi4q@%Ao$x4lw9vLHM!6mn3-b_cebF|n-2vt-zYVF_&sDE--J-P;2WHo z+@n2areE0o$LjvjlV2X7ZU@j+`{*8zq`JR3gKF#EW|#+{nMyo-a>nFFTg&vhyT=b} zDa8+v0(Dgx0yRL@ZXOYIlVSZ0|MFizy0VPW8;AfA5|pe!#j zX}Py^8fl5SyS4g1WSKKtnyP+_PoOwMMwu`(i@Z)diJp~U54*-miOchy7Z35eL>^M z4p<-aIxH4VUZgS783@H%M7P9hX>t{|RU7$n4T(brCG#h9e9p! z+o`i;EGGq3&pF;~5V~eBD}lC)>if$w%Vf}AFxGqO88|ApfHf&Bvu+xdG)@vuF}Yvk z)o;~k-%+0K0g+L`Wala!$=ZV|z$e%>f0%XoLib%)!R^RoS+{!#X?h-6uu zF&&KxORdZU&EwQFITIRLo(7TA3W}y6X{?Y%y2j0It!ekU#<)$qghZtpcS>L3uh`Uj z7GY;6f$9qKynP#oS3$$a{p^{D+0oJQ71`1?OAn_m8)UGZmj3l*ZI)`V-a>MKGGFG< z&^jg#Ok%(hhm>hSrZ5;Qga4u(?^i>GiW_j9%_7M>j(^|Om$#{k+^*ULnEgzW_1gCICtAD^WpC`A z{9&DXkG#01Xo)U$OC(L5Y$DQ|Q4C6CjUKk1UkPj$nXH##J{c8e#K|&{mA*;b$r0E4 zUNo0jthwA(c&N1l=PEe8Rw_8cEl|-eya9z&H3#n`B$t#+aJ03RFMzrV@gowbe8v(c zIFM60^0&lCFO10NU4w@|61xiZ4CVXeaKjd;d?sv52XM*lS8XiVjgWpRB;&U_C0g+`6B5V&w|O6B*_q zsATxL!M}+$He)1eOWECce#eS@2n^xhlB4<_Nn?yCVEQWDs(r`|@2GqLe<#(|&P0U? z$7V5IgpWf09uIf_RazRwC?qEqRaHyL?iiS05UiGesJy%^>-C{{ypTBI&B0-iUYhk> zIk<5xpsuV@g|z(AZD+C-;A!fTG=df1=<%nxy(a(IS+U{ME4ZbDEBtcD_3V=icT6*_ z)>|J?>&6%nvHhZERBtjK+s4xnut*@>GAmA5m*OTp$!^CHTr}vM4n(X1Q*;{e-Rd2BCF-u@1ZGm z!S8hJ6L=Gl4T_SDa7Xx|-{4mxveJg=ctf`BJ*fy!yF6Dz&?w(Q_6B}WQVtNI!BVBC zKfX<>7vd6C96}XAQmF-Jd?1Q4eTfRB3q7hCh0f!(JkdWT5<{iAE#dKy*Jxq&3a1@~ z8C||Dn2mFNyrUV|<-)C^_y7@8c2Fz+2jrae9deBDu;U}tJ{^xAdxCD248(k;dCJ%o z`y3sADe>U%suxwwv~8A1+R$VB=Q?%U?4joI$um;aH+eCrBqpn- z%79D_7rb;R-;-9RTrwi9dPlg8&@tfWhhZ(Vx&1PQ+6(huX`;M9x~LrW~~#3{j0Bh2kDU$}@!fFQej4VGkJv?M4rU^x!RU zEwhu$!CA_iDjFjrJa`aocySDX16?~;+wgav;}Zut6Mg%C4>}8FL?8)Kgwc(Qlj{@#2Pt0?G`$h7P#M+qoXtlV@d}%c&OzO+QYKK`kyXaK{U(O^2DyIXCZlNQjt0^8~8JzNGrIxhj}}M z&~QZlbx%t;MJ(Vux;2tgNKGlAqphLq%pd}JG9uoVHUo?|hN{pLQ6Em%r*+7t^<);X zm~6=qChlNAVXNN*Sow->*4;}T;l;D1I-5T{Bif@4_}=>l`tK;qqDdt5zvisCKhMAH z#r}`)7VW?LZqfdmXQ%zo5bJ00{Xb9^YKrk0Nf|oIW*K@(=`o2Vndz}ZDyk{!u}PVx zzd--+_WC*U{~DH3{?GI64IB+@On&@9X>EUAo&L+G{L^dozaI4C3G#2wr~hseW@K&g zKWs{uHu-9Je!3;4pE>eBltKUXb^*hG8I&413)$J&{D4N%7PcloU6bn%jPxJyQL?g* z9g+YFFEDiE`8rW^laCNzQmi7CTnPfwyg3VDHRAl>h=In6jeaVOP@!-CP60j3+#vpL zEYmh_oP0{-gTe7Or`L6x)6w?77QVi~jD8lWN@3RHcm80iV%M1A!+Y6iHM)05iC64tb$X2lV_%Txk@0l^hZqi^%Z?#- zE;LE0uFx)R08_S-#(wC=dS&}vj6P4>5ZWjhthP=*Hht&TdLtKDR;rXEX4*z0h74FA zMCINqrh3Vq;s%3MC1YL`{WjIAPkVL#3rj^9Pj9Ss7>7duy!9H0vYF%>1jh)EPqvlr6h%R%CxDsk| z!BACz7E%j?bm=pH6Eaw{+suniuY7C9Ut~1cWfOX9KW9=H><&kQlinPV3h9R>3nJvK z4L9(DRM=x;R&d#a@oFY7mB|m8h4692U5eYfcw|QKwqRsshN(q^v$4$)HgPpAJDJ`I zkqjq(8Cd!K!+wCd=d@w%~e$=gdUgD&wj$LQ1r>-E=O@c ze+Z$x{>6(JA-fNVr)X;*)40Eym1TtUZI1Pwwx1hUi+G1Jlk~vCYeXMNYtr)1?qwyg zsX_e*$h?380O00ou?0R@7-Fc59o$UvyVs4cUbujHUA>sH!}L54>`e` zHUx#Q+Hn&Og#YVOuo*niy*GU3rH;%f``nk#NN5-xrZ34NeH$l`4@t);4(+0|Z#I>Y z)~Kzs#exIAaf--65L0UHT_SvV8O2WYeD>Mq^Y6L!Xu8%vnpofG@w!}R7M28?i1*T&zp3X4^OMCY6(Dg<-! zXmcGQrRgHXGYre7GfTJ)rhl|rs%abKT_Nt24_Q``XH{88NVPW+`x4ZdrMuO0iZ0g` z%p}y};~T5gbb9SeL8BSc`SO#ixC$@QhXxZ=B}L`tP}&k?1oSPS=4%{UOHe0<_XWln zwbl5cn(j-qK`)vGHY5B5C|QZd5)W7c@{bNVXqJ!!n$^ufc?N9C-BF2QK1(kv++h!>$QbAjq)_b$$PcJdV+F7hz0Hu@ zqj+}m0qn{t^tD3DfBb~0B36|Q`bs*xs|$i^G4uNUEBl4g;op-;Wl~iThgga?+dL7s zUP(8lMO?g{GcYpDS{NM!UA8Hco?#}eNEioRBHy4`mq!Pd-9@-97|k$hpEX>xoX+dY zDr$wfm^P&}Wu{!%?)U_(%Mn79$(ywvu*kJ9r4u|MyYLI_67U7%6Gd_vb##Nerf@>& z8W11z$$~xEZt$dPG}+*IZky+os5Ju2eRi;1=rUEeIn>t-AzC_IGM-IXWK3^6QNU+2pe=MBn4I*R@A%-iLDCOHTE-O^wo$sL_h{dcPl=^muAQb`_BRm};=cy{qSkui;`WSsj9%c^+bIDQ z0`_?KX0<-=o!t{u(Ln)v>%VGL z0pC=GB7*AQ?N7N{ut*a%MH-tdtNmNC+Yf$|KS)BW(gQJ*z$d{+{j?(e&hgTy^2|AR9vx1Xre2fagGv0YXWqtNkg*v%40v?BJBt|f9wX5 z{QTlCM}b-0{mV?IG>TW_BdviUKhtosrBqdfq&Frdz>cF~yK{P@(w{Vr7z2qKFwLhc zQuogKO@~YwyS9%+d-zD7mJG~@?EFJLSn!a&mhE5$_4xBl&6QHMzL?CdzEnC~C3$X@ zvY!{_GR06ep5;<#cKCSJ%srxX=+pn?ywDwtJ2{TV;0DKBO2t++B(tIO4)Wh`rD13P z4fE$#%zkd=UzOB74gi=-*CuID&Z3zI^-`4U^S?dHxK8fP*;fE|a(KYMgMUo`THIS1f!*6dOI2 zFjC3O=-AL`6=9pp;`CYPTdVX z8(*?V&%QoipuH0>WKlL8A*zTKckD!paN@~hh zmXzm~qZhMGVdQGd=AG8&20HW0RGV8X{$9LldFZYm zE?}`Q3i?xJRz43S?VFMmqRyvWaS#(~Lempg9nTM$EFDP(Gzx#$r)W&lpFKqcAoJh-AxEw$-bjW>`_+gEi z2w`99#UbFZGiQjS8kj~@PGqpsPX`T{YOj`CaEqTFag;$jY z8_{Wzz>HXx&G*Dx<5skhpETxIdhKH?DtY@b9l8$l?UkM#J-Snmts7bd7xayKTFJ(u zyAT&@6cAYcs{PBfpqZa%sxhJ5nSZBPji?Zlf&}#L?t)vC4X5VLp%~fz2Sx<*oN<7` z?ge=k<=X7r<~F7Tvp9#HB{!mA!QWBOf%EiSJ6KIF8QZNjg&x~-%e*tflL(ji_S^sO ztmib1rp09uon}RcsFi#k)oLs@$?vs(i>5k3YN%$T(5Or(TZ5JW9mA6mIMD08=749$ z!d+l*iu{Il7^Yu}H;lgw=En1sJpCKPSqTCHy4(f&NPelr31^*l%KHq^QE>z>Ks_bH zjbD?({~8Din7IvZeJ>8Ey=e;I?thpzD=zE5UHeO|neioJwG;IyLk?xOz(yO&0DTU~ z^#)xcs|s>Flgmp;SmYJ4g(|HMu3v7#;c*Aa8iF#UZo7CvDq4>8#qLJ|YdZ!AsH%^_7N1IQjCro

K7UpUK$>l@ zw`1S}(D?mUXu_C{wupRS-jiX~w=Uqqhf|Vb3Cm9L=T+w91Cu^ z*&Ty%sN?x*h~mJc4g~k{xD4ZmF%FXZNC;oVDwLZ_WvrnzY|{v8hc1nmx4^}Z;yriXsAf+Lp+OFLbR!&Ox?xABwl zu8w&|5pCxmu#$?Cv2_-Vghl2LZ6m7}VLEfR5o2Ou$x02uA-%QB2$c(c1rH3R9hesc zfpn#oqpbKuVsdfV#cv@5pV4^f_!WS+F>SV6N0JQ9E!T90EX((_{bSSFv9ld%I0&}9 zH&Jd4MEX1e0iqDtq~h?DBrxQX1iI0lIs<|kB$Yrh&cpeK0-^K%=FBsCBT46@h#yi!AyDq1V(#V}^;{{V*@T4WJ&U-NTq43w=|K>z8%pr_nC>%C(Wa_l78Ufib$r8Od)IIN=u>417 z`Hl{9A$mI5A(;+-Q&$F&h-@;NR>Z<2U;Y21>>Z;s@0V@SbkMQQj%_;~+qTuQ?c|AV zcWm3XZQHhP&R%QWarS%mJ!9R^&!_)*s(v+VR@I#QrAT}`17Y+l<`b-nvmDNW`De%y zrwTZ9EJrj1AFA>B`1jYDow}~*dfPs}IZMO3=a{Fy#IOILc8F0;JS4x(k-NSpbN@qM z`@aE_e}5{!$v3+qVs7u?sOV(y@1Os*Fgu`fCW9=G@F_#VQ%xf$hj0~wnnP0$hFI+@ zkQj~v#V>xn)u??YutKsX>pxKCl^p!C-o?+9;!Nug^ z{rP!|+KsP5%uF;ZCa5F;O^9TGac=M|=V z_H(PfkV1rz4jl?gJ(ArXMyWT4y(86d3`$iI4^l9`vLdZkzpznSd5Ikfrs8qcSy&>z zTIZgWZGXw0n9ibQxYWE@gI0(3#KA-dAdPcsL_|hg2@~C!VZDM}5;v_Nykfq!*@*Zf zE_wVgx82GMDryKO{U{D>vSzSc%B~|cjDQrt5BN=Ugpsf8H8f1lR4SGo#hCuXPL;QQ z#~b?C4MoepT3X`qdW2dNn& zo8)K}%Lpu>0tQei+{>*VGErz|qjbK#9 zvtd8rcHplw%YyQCKR{kyo6fgg!)6tHUYT(L>B7er5)41iG`j$qe*kSh$fY!PehLcD zWeKZHn<492B34*JUQh=CY1R~jT9Jt=k=jCU2=SL&&y5QI2uAG2?L8qd2U(^AW#{(x zThSy=C#>k+QMo^7caQcpU?Qn}j-`s?1vXuzG#j8(A+RUAY})F@=r&F(8nI&HspAy4 z4>(M>hI9c7?DCW8rw6|23?qQMSq?*Vx?v30U%luBo)B-k2mkL)Ljk5xUha3pK>EEj z@(;tH|M@xkuN?gsz;*bygizwYR!6=(Xgcg^>WlGtRYCozY<rFX2E>kaZo)O<^J7a`MX8Pf`gBd4vrtD|qKn&B)C&wp0O-x*@-|m*0egT=-t@%dD zgP2D+#WPptnc;_ugD6%zN}Z+X4=c61XNLb7L1gWd8;NHrBXwJ7s0ce#lWnnFUMTR& z1_R9Fin4!d17d4jpKcfh?MKRxxQk$@)*hradH2$3)nyXep5Z;B z?yX+-Bd=TqO2!11?MDtG0n(*T^!CIiF@ZQymqq1wPM_X$Iu9-P=^}v7npvvPBu!d$ z7K?@CsA8H38+zjA@{;{kG)#AHME>Ix<711_iQ@WWMObXyVO)a&^qE1GqpP47Q|_AG zP`(AD&r!V^MXQ^e+*n5~Lp9!B+#y3#f8J^5!iC@3Y@P`;FoUH{G*pj*q7MVV)29+j z>BC`a|1@U_v%%o9VH_HsSnM`jZ-&CDvbiqDg)tQEnV>b%Ptm)T|1?TrpIl)Y$LnG_ zzKi5j2Fx^K^PG1=*?GhK;$(UCF-tM~^=Z*+Wp{FSuy7iHt9#4n(sUuHK??@v+6*|10Csdnyg9hAsC5_OrSL;jVkLlf zHXIPukLqbhs~-*oa^gqgvtpgTk_7GypwH><53riYYL*M=Q@F-yEPLqQ&1Sc zZB%w}T~RO|#jFjMWcKMZccxm-SL)s_ig?OC?y_~gLFj{n8D$J_Kw%{r0oB8?@dWzn zB528d-wUBQzrrSSLq?fR!K%59Zv9J4yCQhhDGwhptpA5O5U?Hjqt>8nOD zi{)0CI|&Gu%zunGI*XFZh(ix)q${jT8wnnzbBMPYVJc4HX*9d^mz|21$=R$J$(y7V zo0dxdbX3N#=F$zjstTf*t8vL)2*{XH!+<2IJ1VVFa67|{?LP&P41h$2i2;?N~RA30LV`BsUcj zfO9#Pg1$t}7zpv#&)8`mis3~o+P(DxOMgz-V*(?wWaxi?R=NhtW}<#^Z?(BhSwyar zG|A#Q7wh4OfK<|DAcl9THc-W4*>J4nTevsD%dkj`U~wSUCh15?_N@uMdF^Kw+{agk zJ`im^wDqj`Ev)W3k3stasP`88-M0ZBs7;B6{-tSm3>I@_e-QfT?7|n0D~0RRqDb^G zyHb=is;IwuQ&ITzL4KsP@Z`b$d%B0Wuhioo1CWttW8yhsER1ZUZzA{F*K=wmi-sb#Ju+j z-l@In^IKnb{bQG}Ps>+Vu_W#grNKNGto+yjA)?>0?~X`4I3T@5G1)RqGUZuP^NJCq&^HykuYtMDD8qq+l8RcZNJsvN(10{ zQ1$XcGt}QH-U^WU!-wRR1d--{B$%vY{JLWIV%P4-KQuxxDeJaF#{eu&&r!3Qu{w}0f--8^H|KwE>)ORrcR+2Qf zb})DRcH>k0zWK8@{RX}NYvTF;E~phK{+F;MkIP$)T$93Ba2R2TvKc>`D??#mv9wg$ zd~|-`Qx5LwwsZ2hb*Rt4S9dsF%Cny5<1fscy~)d;0m2r$f=83<->c~!GNyb!U)PA; zq^!`@@)UaG)Ew(9V?5ZBq#c%dCWZrplmuM`o~TyHjAIMh0*#1{B>K4po-dx$Tk-Cq z=WZDkP5x2W&Os`N8KiYHRH#UY*n|nvd(U>yO=MFI-2BEp?x@=N<~CbLJBf6P)}vLS?xJXYJ2^<3KJUdrwKnJnTp{ zjIi|R=L7rn9b*D#Xxr4*R<3T5AuOS+#U8hNlfo&^9JO{VbH!v9^JbK=TCGR-5EWR@ zN8T-_I|&@A}(hKeL4_*eb!1G8p~&_Im8|wc>Cdir+gg90n1dw?QaXcx6Op_W1r=axRw>4;rM*UOpT#Eb9xU1IiWo@h?|5uP zka>-XW0Ikp@dIe;MN8B01a7+5V@h3WN{J=HJ*pe0uwQ3S&MyWFni47X32Q7SyCTNQ z+sR!_9IZa5!>f&V$`q!%H8ci!a|RMx5}5MA_kr+bhtQy{-^)(hCVa@I!^TV4RBi zAFa!Nsi3y37I5EK;0cqu|9MRj<^r&h1lF}u0KpKQD^5Y+LvFEwM zLU@@v4_Na#Axy6tn3P%sD^5P#<7F;sd$f4a7LBMk zGU^RZHBcxSA%kCx*eH&wgA?Qwazm8>9SCSz_!;MqY-QX<1@p$*T8lc?@`ikEqJ>#w zcG``^CoFMAhdEXT9qt47g0IZkaU)4R7wkGs^Ax}usqJ5HfDYAV$!=6?>J6+Ha1I<5 z|6=9soU4>E))tW$<#>F ziZ$6>KJf0bPfbx_)7-}tMINlc=}|H+$uX)mhC6-Hz+XZxsKd^b?RFB6et}O#+>Wmw9Ec9) z{q}XFWp{3@qmyK*Jvzpyqv57LIR;hPXKsrh{G?&dRjF%Zt5&m20Ll?OyfUYC3WRn{cgQ?^V~UAv+5 z&_m#&nIwffgX1*Z2#5^Kl4DbE#NrD&Hi4|7SPqZ}(>_+JMz=s|k77aEL}<=0Zfb)a z%F(*L3zCA<=xO)2U3B|pcTqDbBoFp>QyAEU(jMu8(jLA61-H!ucI804+B!$E^cQQa z)_ERrW3g!B9iLb3nn3dlkvD7KsY?sRvls3QC0qPi>o<)GHx%4Xb$5a3GBTJ(k@`e@ z$RUa^%S15^1oLEmA=sayrP5;9qtf!Z1*?e$ORVPsXpL{jL<6E)0sj&swP3}NPmR%FM?O>SQgN5XfHE< zo(4#Cv11(%Nnw_{_Ro}r6=gKd{k?NebJ~<~Kv0r(r0qe4n3LFx$5%x(BKvrz$m?LG zjLIc;hbj0FMdb9aH9Lpsof#yG$(0sG2%RL;d(n>;#jb!R_+dad+K;Ccw!|RY?uS(a zj~?=&M!4C(5LnlH6k%aYvz@7?xRa^2gml%vn&eKl$R_lJ+e|xsNfXzr#xuh(>`}9g zLHSyiFwK^-p!;p$yt7$F|3*IfO3Mlu9e>Dpx8O`37?fA`cj`C0B-m9uRhJjs^mRp# zWB;Aj6|G^1V6`jg7#7V9UFvnB4((nIwG?k%c7h`?0tS8J3Bn0t#pb#SA}N-|45$-j z$R>%7cc2ebAClXc(&0UtHX<>pd)akR3Kx_cK+n<}FhzmTx!8e9^u2e4%x{>T6pQ`6 zO182bh$-W5A3^wos0SV_TgPmF4WUP-+D25KjbC{y_6W_9I2_vNKwU(^qSdn&>^=*t z&uvp*@c8#2*paD!ZMCi3;K{Na;I4Q35zw$YrW5U@Kk~)&rw;G?d7Q&c9|x<Hg|CNMsxovmfth*|E*GHezPTWa^Hd^F4!B3sF;)? z(NaPyAhocu1jUe(!5Cy|dh|W2=!@fNmuNOzxi^tE_jAtzNJ0JR-avc_H|ve#KO}#S z#a(8secu|^Tx553d4r@3#6^MHbH)vmiBpn0X^29xEv!Vuh1n(Sr5I0V&`jA2;WS|Y zbf0e}X|)wA-Pf5gBZ>r4YX3Mav1kKY(ulAJ0Q*jB)YhviHK)w!TJsi3^dMa$L@^{` z_De`fF4;M87vM3Ph9SzCoCi$#Fsd38u!^0#*sPful^p5oI(xGU?yeYjn;Hq1!wzFk zG&2w}W3`AX4bxoVm03y>ts{KaDf!}b&7$(P4KAMP=vK5?1In^-YYNtx1f#}+2QK@h zeSeAI@E6Z8a?)>sZ`fbq9_snl6LCu6g>o)rO;ijp3|$vig+4t} zylEo7$SEW<_U+qgVcaVhk+4k+C9THI5V10qV*dOV6pPtAI$)QN{!JRBKh-D zk2^{j@bZ}yqW?<#VVuI_27*cI-V~sJiqQv&m07+10XF+#ZnIJdr8t`9s_EE;T2V;B z4UnQUH9EdX%zwh-5&wflY#ve!IWt0UE-My3?L#^Bh%kcgP1q{&26eXLn zTkjJ*w+(|_>Pq0v8{%nX$QZbf)tbJaLY$03;MO=Ic-uqYUmUCuXD>J>o6BCRF=xa% z3R4SK9#t1!K4I_d>tZgE>&+kZ?Q}1qo4&h%U$GfY058s%*=!kac{0Z+4Hwm!)pFLR zJ+5*OpgWUrm0FPI2ib4NPJ+Sk07j(`diti^i#kh&f}i>P4~|d?RFb#!JN)~D@)beox}bw?4VCf^y*`2{4`-@%SFTry2h z>9VBc9#JxEs1+0i2^LR@B1J`B9Ac=#FW=(?2;5;#U$0E0UNag_!jY$&2diQk_n)bT zl5Me_SUvqUjwCqmVcyb`igygB_4YUB*m$h5oeKv3uIF0sk}~es!{D>4r%PC*F~FN3owq5e0|YeUTSG#Vq%&Gk7uwW z0lDo#_wvflqHeRm*}l?}o;EILszBt|EW*zNPmq#?4A+&i0xx^?9obLyY4xx=Y9&^G;xYXYPxG)DOpPg!i_Ccl#3L}6xAAZzNhPK1XaC_~ z!A|mlo?Be*8Nn=a+FhgpOj@G7yYs(Qk(8&|h@_>w8Y^r&5nCqe0V60rRz?b5%J;GYeBqSAjo|K692GxD4` zRZyM2FdI+-jK2}WAZTZ()w_)V{n5tEb@>+JYluDozCb$fA4H)$bzg(Ux{*hXurjO^ zwAxc+UXu=&JV*E59}h3kzQPG4M)X8E*}#_&}w*KEgtX)cU{vm9b$atHa;s>| z+L6&cn8xUL*OSjx4YGjf6{Eq+Q3{!ZyhrL&^6Vz@jGbI%cAM9GkmFlamTbcQGvOlL zmJ?(FI)c86=JEs|*;?h~o)88>12nXlpMR4@yh%qdwFNpct;vMlc=;{FSo*apJ;p}! zAX~t;3tb~VuP|ZW;z$=IHf->F@Ml)&-&Bnb{iQyE#;GZ@C$PzEf6~q}4D>9jic@mTO5x76ulDz@+XAcm35!VSu zT*Gs>;f0b2TNpjU_BjHZ&S6Sqk6V1370+!eppV2H+FY!q*n=GHQ!9Rn6MjY!Jc77A zG7Y!lFp8?TIHN!LXO?gCnsYM-gQxsm=Ek**VmZu7vnuufD7K~GIxfxbsQ@qv2T zPa`tvHB$fFCyZl>3oYg?_wW)C>^_iDOc^B7klnTOoytQH18WkOk)L2BSD0r%xgRSW zQS9elF^?O=_@|58zKLK;(f77l-Zzu}4{fXed2saq!5k#UZAoDBqYQS{sn@j@Vtp|$ zG%gnZ$U|9@u#w1@11Sjl8ze^Co=)7yS(}=;68a3~g;NDe_X^}yJj;~s8xq9ahQ5_r zxAlTMnep*)w1e(TG%tWsjo3RR;yVGPEO4V{Zp?=a_0R#=V^ioQu4YL=BO4r0$$XTX zZfnw#_$V}sDAIDrezGQ+h?q24St0QNug_?{s-pI(^jg`#JRxM1YBV;a@@JQvH8*>> zIJvku74E0NlXkYe_624>znU0J@L<-c=G#F3k4A_)*;ky!C(^uZfj%WB3-*{*B$?9+ zDm$WFp=0(xnt6`vDQV3Jl5f&R(Mp};;q8d3I%Kn>Kx=^;uSVCw0L=gw53%Bp==8Sw zxtx=cs!^-_+i{2OK`Q;913+AXc_&Z5$@z3<)So0CU3;JAv=H?@Zpi~riQ{z-zLtVL z!oF<}@IgJp)Iyz1zVJ42!SPHSkjYNS4%ulVVIXdRuiZ@5Mx8LJS}J#qD^Zi_xQ@>DKDr-_e#>5h3dtje*NcwH_h;i{Sx7}dkdpuW z(yUCjckQsagv*QGMSi9u1`Z|V^}Wjf7B@q%j2DQXyd0nOyqg%m{CK_lAoKlJ7#8M} z%IvR?Vh$6aDWK2W!=i?*<77q&B8O&3?zP(Cs@kapc)&p7En?J;t-TX9abGT#H?TW? ztO5(lPKRuC7fs}zwcUKbRh=7E8wzTsa#Z{a`WR}?UZ%!HohN}d&xJ=JQhpO1PI#>X zHkb>pW04pU%Bj_mf~U}1F1=wxdBZu1790>3Dm44bQ#F=T4V3&HlOLsGH)+AK$cHk6 zia$=$kog?)07HCL*PI6}DRhpM^*%I*kHM<#1Se+AQ!!xyhcy6j7`iDX7Z-2i73_n# zas*?7LkxS-XSqv;YBa zW_n*32D(HTYQ0$feV_Fru1ZxW0g&iwqixPX3=9t4o)o|kOo79V$?$uh?#8Q8e>4e)V6;_(x&ViUVxma+i25qea;d-oK7ouuDsB^ab{ zu1qjQ%`n56VtxBE#0qAzb7lph`Eb-}TYpXB!H-}3Ykqyp`otprp7{VEuW*^IR2n$Fb99*nAtqT&oOFIf z@w*6>YvOGw@Ja?Pp1=whZqydzx@9X4n^2!n83C5{C?G@|E?&$?p*g68)kNvUTJ)I6 z1Q|(#UuP6pj78GUxq11m-GSszc+)X{C2eo-?8ud9sB=3(D47v?`JAa{V(IF zPZQ_0AY*9M97>Jf<o%#O_%Wq}8>YM=q0|tGY+hlXcpE=Z4Od z`NT7Hu2hnvRoqOw@g1f=bv`+nba{GwA$Ak0INlqI1k<9!x_!sL()h?hEWoWrdU3w` zZ%%)VR+Bc@_v!C#koM1p-3v_^L6)_Ktj4HE>aUh%2XZE@JFMOn)J~c`_7VWNb9c-N z2b|SZMR4Z@E7j&q&9(6H3yjEu6HV7{2!1t0lgizD;mZ9$r(r7W5G$ky@w(T_dFnOD z*p#+z$@pKE+>o@%eT(2-p_C}wbQ5s(%Sn_{$HDN@MB+Ev?t@3dPy`%TZ!z}AThZSu zN<1i$siJhXFdjV zP*y|V<`V8t=h#XTRUR~5`c`Z9^-`*BZf?WAehGdg)E2Je)hqFa!k{V(u+(hTf^Yq& zoruUh2(^3pe)2{bvt4&4Y9CY3js)PUHtd4rVG57}uFJL)D(JfSIo^{P=7liFXG zq5yqgof0V8paQcP!gy+;^pp-DA5pj=gbMN0eW=-eY+N8~y+G>t+x}oa!5r>tW$xhI zPQSv=pi;~653Gvf6~*JcQ%t1xOrH2l3Zy@8AoJ+wz@daW@m7?%LXkr!bw9GY@ns3e zSfuWF_gkWnesv?s3I`@}NgE2xwgs&rj?kH-FEy82=O8`+szN ziHch`vvS`zNfap14!&#i9H@wF7}yIPm=UB%(o(}F{wsZ(wA0nJ2aD^@B41>>o-_U6 zUqD~vdo48S8~FTb^+%#zcbQiiYoDKYcj&$#^;Smmb+Ljp(L=1Kt_J!;0s%1|JK}Wi z;={~oL!foo5n8=}rs6MmUW~R&;SIJO3TL4Ky?kh+b2rT9B1Jl4>#Uh-Bec z`Hsp<==#UEW6pGPhNk8H!!DUQR~#F9jEMI6T*OWfN^Ze&X(4nV$wa8QUJ>oTkruH# zm~O<`J7Wxseo@FqaZMl#Y(mrFW9AHM9Kb|XBMqaZ2a)DvJgYipkDD_VUF_PKd~dT7 z#02}bBfPn9a!X!O#83=lbJSK#E}K&yx-HI#T6ua)6o0{|={*HFusCkHzs|Fn&|C3H zBck1cmfcWVUN&i>X$YU^Sn6k2H;r3zuXbJFz)r5~3$d$tUj(l1?o={MM){kjgqXRO zc5R*#{;V7AQh|G|)jLM@wGAK&rm2~@{Pewv#06pHbKn#wL0P6F1!^qw9g&cW3Z=9} zj)POhOlwsh@eF=>z?#sIs*C-Nl(yU!#DaiaxhEs#iJqQ8w%(?+6lU02MYSeDkr!B- zPjMv+on6OLXgGnAtl(ao>|X2Y8*Hb}GRW5}-IzXnoo-d0!m4Vy$GS!XOLy>3_+UGs z2D|YcQx@M#M|}TDOetGi{9lGo9m-=0-^+nKE^*?$^uHkxZh}I{#UTQd;X!L+W@jm( zDg@N4+lUqI92o_rNk{3P>1gxAL=&O;x)ZT=q1mk0kLlE$WeWuY_$0`0jY-Kkt zP*|m3AF}Ubd=`<>(Xg0har*_@x2YH}bn0Wk*OZz3*e5;Zc;2uBdnl8?&XjupbkOeNZsNh6pvsq_ydmJI+*z**{I{0K)-;p1~k8cpJXL$^t!-`E}=*4G^-E8>H!LjTPxSx zcF+cS`ommfKMhNSbas^@YbTpH1*RFrBuATUR zt{oFWSk^$xU&kbFQ;MCX22RAN5F6eq9UfR$ut`Jw--p2YX)A*J69m^!oYfj2y7NYcH6&r+0~_sH^c^nzeN1AU4Ga7=FlR{S|Mm~MpzY0$Z+p2W(a={b-pR9EO1Rs zB%KY|@wLcAA@)KXi!d2_BxrkhDn`DT1=Dec}V!okd{$+wK z4E{n8R*xKyci1(CnNdhf$Dp2(Jpof0-0%-38X=Dd9PQgT+w%Lshx9+loPS~MOm%ZT zt%2B2iL_KU_ita%N>xjB!#71_3=3c}o zgeW~^U_ZTJQ2!PqXulQd=3b=XOQhwATK$y(9$#1jOQ4}4?~l#&nek)H(04f(Sr=s| zWv7Lu1=%WGk4FSw^;;!8&YPM)pQDCY9DhU`hMty1@sq1=Tj7bFsOOBZOFlpR`W>-J$-(kezWJj;`?x-v>ev{*8V z8p|KXJPV$HyQr1A(9LVrM47u-XpcrIyO`yWvx1pVYc&?154aneRpLqgx)EMvRaa#|9?Wwqs2+W8n5~79G z(}iCiLk;?enn}ew`HzhG+tu+Ru@T+K5juvZN)wY;x6HjvqD!&!)$$;1VAh~7fg0K| zEha#aN=Yv|3^~YFH}cc38ovVb%L|g@9W6fo(JtT6$fa?zf@Ct88e}m?i)b*Jgc{fl zExfdvw-BYDmH6>(4QMt#p0;FUIQqkhD}aH?a7)_%JtA~soqj{ppP_82yi9kaxuK>~ ze_)Zt>1?q=ZH*kF{1iq9sr*tVuy=u>Zev}!gEZx@O6-fjyu9X00gpIl-fS_pzjpqJ z1yqBmf9NF!jaF<+YxgH6oXBdK)sH(>VZ)1siyA$P<#KDt;8NT*l_0{xit~5j1P)FN zI8hhYKhQ)i z37^aP13B~u65?sg+_@2Kr^iWHN=U;EDSZ@2W2!5ALhGNWXnFBY%7W?1 z=HI9JzQ-pLKZDYTv<0-lt|6c-RwhxZ)mU2Os{bsX_i^@*fKUj8*aDO5pks=qn3Dv6 zwggpKLuyRCTVPwmw1r}B#AS}?X7b837UlXwp~E2|PJw2SGVueL7){Y&z!jL!XN=0i zU^Eig`S2`{+gU$68aRdWx?BZ{sU_f=8sn~>s~M?GU~`fH5kCc; z8ICp+INM3(3{#k32RZdv6b9MQYdZXNuk7ed8;G?S2nT+NZBG=Tar^KFl2SvhW$bGW#kdWL-I)s_IqVnCDDM9fm8g;P;8 z7t4yZn3^*NQfx7SwmkzP$=fwdC}bafQSEF@pd&P8@H#`swGy_rz;Z?Ty5mkS%>m#% zp_!m9e<()sfKiY(nF<1zBz&&`ZlJf6QLvLhl`_``%RW&{+O>Xhp;lwSsyRqGf=RWd zpftiR`={2(siiPAS|p}@q=NhVc0ELprt%=fMXO3B)4ryC2LT(o=sLM7hJC!}T1@)E zA3^J$3&1*M6Xq>03FX`R&w*NkrZE?FwU+Muut;>qNhj@bX17ZJxnOlPSZ=Zeiz~T_ zOu#yc3t6ONHB;?|r4w+pI)~KGN;HOGC)txxiUN8#mexj+W(cz%9a4sx|IRG=}ia zuEBuba3AHsV2feqw-3MvuL`I+2|`Ud4~7ZkN=JZ;L20|Oxna5vx1qbIh#k2O4$RQF zo`tL()zxaqibg^GbB+BS5#U{@K;WWQj~GcB1zb}zJkPwH|5hZ9iH2308!>_;%msji zJHSL~s)YHBR=Koa1mLEOHos*`gp=s8KA-C zu0aE+W!#iJ*0xqKm3A`fUGy#O+X+5W36myS>Uh2!R*s$aCU^`K&KKLCCDkejX2p=5 z%o7-fl03x`gaSNyr?3_JLv?2RLS3F*8ub>Jd@^Cc17)v8vYEK4aqo?OS@W9mt%ITJ z9=S2%R8M){CugT@k~~0x`}Vl!svYqX=E)c_oU6o}#Hb^%G1l3BudxA{F*tbjG;W_>=xV73pKY53v%>I)@D36I_@&p$h|Aw zonQS`07z_F#@T-%@-Tb|)7;;anoD_WH>9ewFy(ZcEOM$#Y)8>qi7rCnsH9GO-_7zF zu*C87{Df1P4TEOsnzZ@H%&lvV(3V@;Q!%+OYRp`g05PjY^gL$^$-t0Y>H*CDDs?FZly*oZ&dxvsxaUWF!{em4{A>n@vpXg$dwvt@_rgmHF z-MER`ABa8R-t_H*kv>}CzOpz;!>p^^9ztHMsHL|SRnS<-y5Z*r(_}c4=fXF`l^-i}>e7v!qs_jv zqvWhX^F=2sDNWA9c@P0?lUlr6ecrTKM%pNQ^?*Lq?p-0~?_j50xV%^(+H>sMul#Tw zeciF*1=?a7cI(}352%>LO96pD+?9!fNyl^9v3^v&Y4L)mNGK0FN43&Xf8jUlxW1Bw zyiu2;qW-aGNhs=zbuoxnxiwZ3{PFZM#Kw)9H@(hgX23h(`Wm~m4&TvoZoYp{plb^> z_#?vXcxd>r7K+1HKJvhed>gtK`TAbJUazUWQY6T~t2af%#<+Veyr%7-#*A#@&*;@g58{i|E%6yC_InGXCOd{L0;$)z#?n7M`re zh!kO{6=>7I?*}czyF7_frt#)s1CFJ_XE&VrDA?Dp3XbvF{qsEJgb&OLSNz_5g?HpK z9)8rsr4JN!Af3G9!#Qn(6zaUDqLN(g2g8*M)Djap?WMK9NKlkC)E2|-g|#-rp%!Gz zAHd%`iq|81efi93m3yTBw3g0j#;Yb2X{mhRAI?&KDmbGqou(2xiRNb^sV}%%Wu0?< z?($L>(#BO*)^)rSgyNRni$i`R4v;GhlCZ8$@e^ROX(p=2_v6Y!%^As zu022)fHdv_-~Yu_H6WVPLpHQx!W%^6j)cBhS`O3QBW#x(eX54d&I22op(N59b*&$v zFiSRY6rOc^(dgSV1>a7-5C;(5S5MvKcM2Jm-LD9TGqDpP097%52V+0>Xqq!! zq4e3vj53SE6i8J`XcQB|MZPP8j;PAOnpGnllH6#Ku~vS42xP*Nz@~y%db7Xi8s09P z1)e%8ys6&M8D=Dt6&t`iKG_4X=!kgRQoh%Z`dc&mlOUqXk-k`jKv9@(a^2-Upw>?< zt5*^DV~6Zedbec4NVl($2T{&b)zA@b#dUyd>`2JC0=xa_fIm8{5um zr-!ApXZhC8@=vC2WyxO|!@0Km)h8ep*`^he92$@YwP>VcdoS5OC^s38e#7RPsg4j+ zbVGG}WRSET&ZfrcR(x~k8n1rTP%CnfUNKUonD$P?FtNFF#cn!wEIab-;jU=B1dHK@ z(;(yAQJ`O$sMn>h;pf^8{JISW%d+@v6@CnXh9n5TXGC}?FI9i-D0OMaIg&mAg=0Kn zNJ7oz5*ReJukD55fUsMuaP+H4tDN&V9zfqF@ zr=#ecUk9wu{0;!+gl;3Bw=Vn^)z$ahVhhw)io!na&9}LmWurLb0zubxK=UEnU*{5P z+SP}&*(iBKSO4{alBHaY^)5Q=mZ+2OwIooJ7*Q5XJ+2|q`9#f?6myq!&oz?klihLq z4C)$XP!BNS0G_Z1&TM>?Jk{S~{F3n83ioli=IO6f%wkvCl(RFFw~j0tb{GvXTx>*sB0McY0s&SNvj4+^h`9nJ_wM>F!Uc>X}9PifQekn0sKI2SAJP!a4h z5cyGTuCj3ZBM^&{dRelIlT^9zcfaAuL5Y~bl!ppSf`wZbK$z#6U~rdclk``e+!qhe z6Qspo*%<)eu6?C;Bp<^VuW6JI|Ncvyn+LlSl;Mp22Bl7ARQ0Xc24%29(ZrdsIPw&-=yHQ7_Vle|5h>AST0 zUGX2Zk34vp?U~IHT|;$U86T+UUHl_NE4m|}>E~6q``7hccCaT^#y+?wD##Q%HwPd8 zV3x4L4|qqu`B$4(LXqDJngNy-{&@aFBvVsywt@X^}iH7P%>bR?ciC$I^U-4Foa`YKI^qDyGK7k%E%c_P=yzAi`YnxGA%DeNd++j3*h^ z=rn>oBd0|~lZ<6YvmkKY*ZJlJ;Im0tqgWu&E92eqt;+NYdxx`eS(4Hw_Jb5|yVvBg z*tbdY^!AN;luEyN4VRhS@-_DC{({ziH{&Z}iGElSV~qvT>L-8G%+yEL zX#MFOhj{InyKG=mvW-<1B@c-}x$vA(nU?>S>0*eN#!SLzQ)Ex7fvQ)S4D<8|I#N$3 zT5Ei`Z?cxBODHX8(Xp73v`IsAYC@9b;t}z0wxVuQSY1J^GRwDPN@qbM-ZF48T$GZ< z8WU+;Pqo?{ghI-KZ-i*ydXu`Ep0Xw^McH_KE9J0S7G;x8Fe`DVG?j3Pv=0YzJ}yZR z%2=oqHiUjvuk0~Ca>Kol4CFi0_xQT~;_F?=u+!kIDl-9g`#ZNZ9HCy17Ga1v^Jv9# z{T4Kb1-AzUxq*MutfOWWZgD*HnFfyYg0&e9f(5tZ>krPF6{VikNeHoc{linPPt#Si z&*g>(c54V8rT_AX!J&bNm-!umPvOR}vDai#`CX___J#=zeB*{4<&2WpaDncZsOkp* zsg<%@@rbrMkR_ux9?LsQxzoBa1s%$BBn6vk#{&&zUwcfzeCBJUwFYSF$08qDsB;gWQN*g!p8pxjofWbqNSZOEKOaTx@+* zwdt5*Q47@EOZ~EZL9s?1o?A%9TJT=Ob_13yyugvPg*e&ZU(r6^k4=2+D-@n=Hv5vu zSXG|hM(>h9^zn=eQ=$6`JO&70&2|%V5Lsx>)(%#;pcOfu>*nk_3HB_BNaH$`jM<^S zcSftDU1?nL;jy)+sfonQN}(}gUW?d_ikr*3=^{G)=tjBtEPe>TO|0ddVB zTklrSHiW+!#26frPXQQ(YN8DG$PZo?(po(QUCCf_OJC`pw*uey00%gmH!`WJkrKXj2!#6?`T25mTu9OJp2L8z3! z=arrL$ZqxuE{%yV)14Kd>k}j7pxZ6#$Dz8$@WV5p8kTqN<-7W)Q7Gt2{KoOPK_tZ| zf2WG~O5@{qPI+W<4f_;reuFVdO^5`ADC1!JQE|N`s3cq@(0WB!n0uh@*c{=LAd;~} zyGK@hbF-Oo+!nN)@i*O(`@FA#u?o=~e{`4O#5}z&=UkU*50fOrzi11D^&FOqe>wii z?*k+2|EcUs;Gx{!@KBT~>PAwLrIDT7Th=Utu?~?np@t^gFs?zgX=D${RwOY^WGh-+ z+#4$066ISh8eYW#FXWp~S`<*%O^ZuItL1Tyqt8#tZ zY120E;^VG`!lZn&3sPd$RkdHpU#|w+bYV)pJC|SH9g%|5IkxVTQcBA4CL0}$&}ef@ zW^Vtj%M;;_1xxP9x#ex17&4N*{ksO*_4O}xYu(p*JkL#yr}@7b)t5X?%CY<+s5_MJ zuiqt+N_;A(_)%lumoyRFixWa-M7qK_9s6<1X?JDa9fP!+_6u~~M$5L=ipB=7(j#f< zZ34J%=bs549%~_mA(|={uZNs_0?o7;-LBP(ZRnkd{-^|2|=4vUTmtByHL8 zEph`(LSEzQj68a+`d$V<45J7cyv^#|^|%fD#si1Nx!4NW*`l*{->HEWNh6-|g>-=r zXmQ|-i}Ku$ndUeHQ^&ieT!Lf}vf6GaqW9$DJ2NWrqwPY%%4nip$@vK$nRp*_C-v<| zuKz~ZyN&<%!NS26&x?jhy+@awJipMQ-8(X4#Ae5??U<1QMt1l9R=w9fAnEF}NYu$2 z>6}Vkc zIb*A?G*z8^IvibmBKn_u^5&T_1oey0gZS2~obf(#xk=erZGTEdQnt3DMGM+0oPwss zj5zXD;(oWhB_T@~Ig#9@v)AKtXu3>Inmgf@A|-lD-1U>cNyl3h?ADD9)GG4}zUGPk zZzaXe!~Kf?<~@$G?Uql3t8jy9{2!doq4=J}j9ktTxss{p6!9UdjyDERlA*xZ!=Q)KDs5O)phz>Vq3BNGoM(H|=1*Q4$^2fTZw z(%nq1P|5Rt81}SYJpEEzMPl5VJsV5&4e)ZWKDyoZ>1EwpkHx-AQVQc8%JMz;{H~p{=FXV>jIxvm4X*qv52e?Y-f%DJ zxEA165GikEASQ^fH6K#d!Tpu2HP{sFs%E=e$gYd$aj$+xue6N+Wc(rAz~wUsk2`(b z8Kvmyz%bKQxpP}~baG-rwYcYCvkHOi zlkR<=>ZBTU*8RF_d#Bl@zZsRIhx<%~Z@Z=ik z>adw3!DK(8R|q$vy{FTxw%#xliD~6qXmY^7_9kthVPTF~Xy1CfBqbU~?1QmxmU=+k z(ggxvEuA;0e&+ci-zQR{-f7aO{O(Pz_OsEjLh_K>MbvoZ4nxtk5u{g@nPv)cgW_R} z9}EA4K4@z0?7ue}Z(o~R(X&FjejUI2g~08PH1E4w>9o{)S(?1>Z0XMvTb|;&EuyOE zGvWNpYX)Nv<8|a^;1>bh#&znEcl-r!T#pn= z4$?Yudha6F%4b>*8@=BdtXXY4N+`U4Dmx$}>HeVJk-QdTG@t!tVT#0(LeV0gvqyyw z2sEp^9eY0N`u10Tm4n8No&A=)IeEC|gnmEXoNSzu!1<4R<%-9kY_8~5Ej?zRegMn78wuMs#;i&eUA0Zk_RXQ3b&TT} z;SCI=7-FUB@*&;8|n>(_g^HGf3@QODE3LpmX~ELnymQm{Sx9xrKS zK29p~?v@R$0=v6Dr5aW>-!{+h@?Q58|Kz8{{W`%J+lDAdb&M5VHrX_mDY;1-JLnf)ezmPau$)1;=`-FU=-r-83tX=C`S#}GZufju zQ>sXNT0Ny=k@nc%cFnvA_i4SC)?_ORXHq8B4D%el1uPX`c~uG#S1M7C+*MMqLw78E zhY2dI8@+N^qrMI1+;TUda(vGqGSRyU{Fnm`aqrr7bz42c5xsOO-~oZpkzorD1g}Y<6rk&3>PsSGy}W?MtqFky@A(X# zIuNZK0cK?^=;PUAu>j0#HtjbHCV*6?jzA&OoE$*Jlga*}LF`SF?WLhv1O|zqC<>*> zYB;#lsYKx0&kH@BFpW8n*yDcc6?;_zaJs<-jPSkCsSX-!aV=P5kUgF@Nu<{a%#K*F z134Q{9|YX7X(v$62_cY3^G%t~rD>Q0z@)1|zs)vjJ6Jq9;7#Ki`w+eS**En?7;n&7 zu==V3T&eFboN3ZiMx3D8qYc;VjFUk_H-WWCau(VFXSQf~viH0L$gwD$UfFHqNcgN`x}M+YQ6RnN<+@t>JUp#)9YOkqst-Ga?{FsDpEeX0(5v{0J~SEbWiL zXC2}M4?UH@u&|;%0y`eb33ldo4~z-x8zY!oVmV=c+f$m?RfDC35mdQ2E>Pze7KWP- z>!Bh<&57I+O_^s}9Tg^k)h7{xx@0a0IA~GAOt2yy!X%Q$1rt~LbTB6@Du!_0%HV>N zlf)QI1&gvERKwso23mJ!Ou6ZS#zCS5W`gxE5T>C#E|{i<1D35C222I33?Njaz`On7 zi<+VWFP6D{e-{yiN#M|Jgk<44u1TiMI78S5W`Sdb5f+{zu34s{CfWN7a3Cf^@L%!& zN$?|!!9j2c)j$~+R6n#891w-z8(!oBpL2K=+%a$r2|~8-(vQj5_XT`<0Ksf;oP+tz z9CObS!0m)Tgg`K#xBM8B(|Z)Wb&DYL{WTYv`;A=q6~Nnx2+!lTIXtj8J7dZE!P_{z z#f8w6F}^!?^KE#+ZDv+xd5O&3EmomZzsv?>E-~ygGum45fk!SBN&|eo1rKw^?aZJ4 E2O(~oYXATM literal 0 HcmV?d00001 diff --git a/batch/jvm-entryway/gradle/wrapper/gradle-wrapper.properties b/batch/jvm-entryway/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000..fae08049a6f --- /dev/null +++ b/batch/jvm-entryway/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/batch/jvm-entryway/gradlew b/batch/jvm-entryway/gradlew new file mode 100755 index 00000000000..4f906e0c811 --- /dev/null +++ b/batch/jvm-entryway/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/batch/jvm-entryway/gradlew.bat b/batch/jvm-entryway/gradlew.bat new file mode 100644 index 00000000000..ac1b06f9382 --- /dev/null +++ b/batch/jvm-entryway/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/batch/src/main/java/is/hail/JVMEntryway.java b/batch/jvm-entryway/src/main/java/is/hail/JVMEntryway.java similarity index 83% rename from batch/src/main/java/is/hail/JVMEntryway.java rename to batch/jvm-entryway/src/main/java/is/hail/JVMEntryway.java index 77cb6efb358..a70001ad24c 100644 --- a/batch/src/main/java/is/hail/JVMEntryway.java +++ b/batch/jvm-entryway/src/main/java/is/hail/JVMEntryway.java @@ -1,5 +1,6 @@ package is.hail; +import is.hail.QoBOutputStreamManager; import java.io.*; import java.lang.reflect.*; import java.net.*; @@ -8,8 +9,12 @@ import java.util.*; import java.util.concurrent.*; import org.newsclub.net.unix.*; +import org.apache.logging.log4j.*; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.config.Configurator; class JVMEntryway { + private static final Logger log = LogManager.getLogger(JVMEntryway.class); // this will initialize log4j which is required for us to access the QoBAppender in main private static final HashMap classLoaders = new HashMap<>(); public static String throwableToString(Throwable t) throws IOException { @@ -57,9 +62,11 @@ public static void main(String[] args) throws Exception { System.err.println("reading " + i + ": " + realArgs[i]); } - assert realArgs.length >= 2; + assert realArgs.length >= 4; String classPath = realArgs[0]; String mainClass = realArgs[1]; + String scratchDir = realArgs[2]; + String logFile = realArgs[3]; ClassLoader cl = classLoaders.get(classPath); if (cl == null) { @@ -87,6 +94,13 @@ public static void main(String[] args) throws Exception { Method main = klass.getDeclaredMethod("main", String[].class); System.err.println("main method got"); + QoBOutputStreamManager.changeFileInAllAppenders(logFile); + log.info("is.hail.JVMEntryway received arguments:"); + for (int i = 0; i < nRealArgs; ++i) { + log.info(i + ": " + realArgs[i]); + } + log.info("Yielding control to the QoB Job."); + CompletionService gather = new ExecutorCompletionService(executor); Future mainThread = null; Future shouldCancelThread = null; @@ -104,8 +118,12 @@ public void run() { } main.invoke(null, (Object) mainArgs); } catch (IllegalAccessException | InvocationTargetException e) { + log.error("QoB Job threw an exception.", e); throw new RuntimeException(e); + } catch (Exception e) { + log.error("QoB Job threw an exception.", e); } finally { + QoBOutputStreamManager.flushAllAppenders(); Thread.currentThread().setContextClassLoader(oldClassLoader); } } @@ -118,8 +136,12 @@ public void run() { int i = in.readInt(); assert i == 0 : i; } catch (IOException e) { + log.error("Exception encountered in QoB cancel thread.", e); throw new RuntimeException(e); + } catch (Exception e) { + log.error("Exception encountered in QoB cancel thread.", e); } finally { + QoBOutputStreamManager.flushAllAppenders(); Thread.currentThread().setContextClassLoader(oldClassLoader); } } @@ -127,6 +149,13 @@ public void run() { completedThread = gather.take(); } catch (Throwable t) { entrywayException = t; + } finally { + QoBOutputStreamManager.flushAllAppenders(); + LoggerContext context = (LoggerContext) LogManager.getContext(false); + ClassLoader loader = JVMEntryway.class.getClassLoader(); + URL url = loader.getResource("log4j2.properties"); + System.err.println("reconfiguring logging " + url.toString()); + context.setConfigLocation(url.toURI()); // this will force a reconfiguration } if (entrywayException != null) { diff --git a/batch/jvm-entryway/src/main/resources/log4j2.properties b/batch/jvm-entryway/src/main/resources/log4j2.properties new file mode 100644 index 00000000000..73bfb83069f --- /dev/null +++ b/batch/jvm-entryway/src/main/resources/log4j2.properties @@ -0,0 +1,13 @@ +# DO NOT DELETE THIS LINE, it will prevent log4j2 from finding QoBAppender +packages=is.hail + +appenders = QoBAppender +appender.QoBAppender.type=QoBAppender +appender.QoBAppender.name=QoBAppender +appender.QoBAppender.layout.type=PatternLayout +appender.QoBAppender.layout.pattern=%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1}: %p: %m%n + +# configure root logger to append here +rootLogger.level = info +rootLogger.appenderRefs = QoBAppender +rootLogger.appenderRef.QoBAppender.ref = QoBAppender diff --git a/batch/jvm-entryway/src/main/scala/is/hail/QoBAppender.scala b/batch/jvm-entryway/src/main/scala/is/hail/QoBAppender.scala new file mode 100644 index 00000000000..b1fef45b217 --- /dev/null +++ b/batch/jvm-entryway/src/main/scala/is/hail/QoBAppender.scala @@ -0,0 +1,83 @@ +// Do not move this to a different package without updating packages= in log4j2.properties +package is.hail + +import java.io._ +import java.util.concurrent.TimeUnit +import org.apache.logging.log4j.core._ +import org.apache.logging.log4j.core.layout._ +import org.apache.logging.log4j.core.appender._ +import org.apache.logging.log4j.core.config._ +import org.apache.logging.log4j.core.config.plugins._ +import scala.collection.mutable + +object QoBOutputStreamManager { + private var _instances: mutable.Map[Layout[_], QoBOutputStreamManager] = mutable.Map() + + def getInstance(layout: Layout[_]): QoBOutputStreamManager = synchronized { + _instances.getOrElseUpdate(layout, new QoBOutputStreamManager(layout)) + } + + def changeFileInAllAppenders(newFilename: String): Unit = { + _instances.values.foreach(_.changeFile(newFilename)) + } + + def flushAllAppenders(): Unit = { + _instances.values.foreach(_.flush()) + } +} + +class QoBOutputStreamManager(layout: Layout[_]) extends OutputStreamManager( + null, + "QoBOutputStreamManager", + layout, + true +) { + private[this] var filename: String = null + + override def createOutputStream(): OutputStream = { + assert(filename != null) + new BufferedOutputStream(new FileOutputStream(filename)) + } + + override def close(): Unit = { + super.close() + QoBOutputStreamManager._instances.remove(layout) + } + + def changeFile(newFilename: String): Unit = { + if (hasOutputStream()) { + closeOutputStream() + } + filename = newFilename + setOutputStream(createOutputStream()) + } +} + +object QoBAppender { + @PluginFactory + def createAppender( + @PluginAttribute("name") name: String, + @PluginAttribute("ignoreExceptions") ignoreExceptions: Boolean, + @PluginElement("Layout") layout: Layout[_], + @PluginElement("Filters") filter: Filter + ): QoBAppender = { + return new QoBAppender(name, ignoreExceptions, layout, filter) + } +} + +@Plugin(name = "QoBAppender", category = "Core", elementType = "appender", printObject = true) +class QoBAppender( + name: String, + ignoreExceptions: Boolean, + layout: Layout[_], + filter: Filter +) extends AbstractOutputStreamAppender[QoBOutputStreamManager]( + name, + layout, + filter, + ignoreExceptions, + false, + Array[Property](), + QoBOutputStreamManager.getInstance(layout) +) { +} diff --git a/build.yaml b/build.yaml index 9fc4c8e6ab8..ea1e0eff92c 100644 --- a/build.yaml +++ b/build.yaml @@ -836,29 +836,24 @@ steps: - hail_build_image - merge_code - kind: runImage - name: compile_batch_worker_jvm_entryway + name: jvm_entryway_jar image: - valueFrom: base_image.image + valueFrom: hail_build_image.image script: | set -ex - cd /io - - curl -sSfL https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar \ - > junixsocket-selftest-2.3.3-jar-with-dependencies.jar + cd /io/batch/jvm-entryway - javac -cp junixsocket-selftest-2.3.3-jar-with-dependencies.jar \ - batch/src/main/java/is/hail/JVMEntryway.java + chmod 755 ./gradlew + ./gradlew shadowJar inputs: - from: /repo/batch to: /io/batch outputs: - - from: /io/junixsocket-selftest-2.3.3-jar-with-dependencies.jar - to: /junixsocket-selftest-2.3.3-jar-with-dependencies.jar - - from: /io/batch/src/main/java/is/hail - to: /batch-jvm-entryway-classes + - from: /io/batch/jvm-entryway/build/libs/jvm-entryway.jar + to: /jvm-entryway.jar dependsOn: - - base_image + - hail_build_image - merge_code - kind: buildImage2 name: batch_worker_image @@ -886,16 +881,14 @@ steps: to: /io/repo/web_common - from: /repo/letsencrypt/subdomains.txt to: /repo/letsencrypt/subdomains.txt - - from: /batch-jvm-entryway-classes - to: /io/repo/batch/src/main/java/is/hail - - from: /junixsocket-selftest-2.3.3-jar-with-dependencies.jar - to: /io/repo/batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar + - from: /jvm-entryway.jar + to: /io/repo/batch/jvm-entryway/build/libs/jvm-entryway.jar - from: /hail_version to: /io/repo/hail_version dependsOn: - merge_code - hail_ubuntu_image - - compile_batch_worker_jvm_entryway + - jvm_entryway_jar - kind: buildImage2 name: memory_image dockerFile: /io/repo/memory/Dockerfile diff --git a/hail/src/main/scala/is/hail/backend/service/Main.scala b/hail/src/main/scala/is/hail/backend/service/Main.scala index 94db679c2bc..49440f945ed 100644 --- a/hail/src/main/scala/is/hail/backend/service/Main.scala +++ b/hail/src/main/scala/is/hail/backend/service/Main.scala @@ -9,25 +9,7 @@ object Main { val WORKER = "worker" val DRIVER = "driver" - def configureLogging(logFile: String): Unit = { - val logProps = new Properties() - - logProps.put("log4j.rootLogger", "INFO, logfile") - logProps.put("log4j.appender.logfile", "org.apache.log4j.FileAppender") - logProps.put("log4j.appender.logfile.append", true.toString) - logProps.put("log4j.appender.logfile.file", logFile) - logProps.put("log4j.appender.logfile.threshold", "INFO") - logProps.put("log4j.appender.logfile.layout", "org.apache.log4j.PatternLayout") - logProps.put("log4j.appender.logfile.layout.ConversionPattern", HailContext.logFormat) - - LogManager.resetConfiguration() - PropertyConfigurator.configure(logProps) - } - def main(argv: Array[String]): Unit = { - val logFile = argv(1) - configureLogging(logFile) - argv(3) match { case WORKER => Worker.main(argv) case DRIVER => ServiceBackendSocketAPI2.main(argv) From 99c8062fa96214c943b29937c99da509afc02ec4 Mon Sep 17 00:00:00 2001 From: Dan King Date: Tue, 2 May 2023 13:21:56 -0400 Subject: [PATCH 13/26] [batch] xfail highcpu cheapest test (#12959) See https://github.com/hail-is/hail/issues/12958. --------- Co-authored-by: Daniel Goldstein --- batch/test/test_batch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/batch/test/test_batch.py b/batch/test/test_batch.py index 7080ca0cea6..3c3b5783cf2 100644 --- a/batch/test/test_batch.py +++ b/batch/test/test_batch.py @@ -11,7 +11,7 @@ from hailtop.batch.backend import HAIL_GENETICS_HAILTOP_IMAGE from hailtop.batch_client.client import BatchClient from hailtop.config import get_deploy_config, get_user_config -from hailtop.test_utils import skip_in_azure +from hailtop.test_utils import fails_in_azure, skip_in_azure from hailtop.utils import external_requests_client_session, retry_response_returning_functions, sync_sleep_and_backoff from .failure_injecting_client_session import FailureInjectingClientSession @@ -145,6 +145,7 @@ def test_invalid_resource_requests(client: BatchClient): bb.submit() +@fails_in_azure # https://github.com/hail-is/hail/issues/12958 def test_out_of_memory(client: BatchClient): bb = create_batch(client) resources = {'cpu': '0.25', 'memory': '10M', 'storage': '10Gi'} @@ -1007,6 +1008,7 @@ def test_pool_highcpu_instance(client: BatchClient): assert 'highcpu' in status['status']['worker'], str((status, b.debug_info())) +@fails_in_azure # https://github.com/hail-is/hail/issues/12958 def test_pool_highcpu_instance_cheapest(client: BatchClient): bb = create_batch(client) resources = {'cpu': '0.25', 'memory': '50Mi'} From bb7c332b1c474b1ac8c7f906277d0146958b2d0c Mon Sep 17 00:00:00 2001 From: Dan King Date: Tue, 2 May 2023 17:08:43 -0400 Subject: [PATCH 14/26] [query] Python 3.8, Numpy 1.24.2, Bokeh 3.x.x, fixes to plots.py, fixes to req generation (#12929) CHANGELOG: Hail no longer officially supports Python 3.7. Combines https://github.com/hail-is/hail/pull/12927 and https://github.com/hail-is/hail/pull/12908. Many changes. All seem to be necessary together. --- I fixed any new mypy errors or deprecation warnings. I also cleaned up plots.py (which isn't CI'd by mypy) because I was in there and it was a mess. I unified all our pip-tools versions. Require pandas 2 now. That requires Bokeh 3.x.x. Fix the pinned-requirements.txt dependencies so they reflect the actual necessary runtime harmony. Upgraded Sphinx. The method names lose their fixed-width styling but I think it looks fine. Version policy added. Updating everything means Jinja2 can jump to the latest version too. Numpy deprecated some of its types, using `_` gets rid of the warning. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Makefile | 6 +- auth/pinned-requirements.txt | 10 +- batch/Dockerfile.worker | 4 +- batch/pinned-requirements.txt | 29 +- batch/requirements.txt | 2 +- benchmark/python/setup.py | 2 +- build.yaml | 2 +- ci/Dockerfile.ci-utils | 2 +- ci/pinned-requirements.txt | 17 +- dev-docs/compiler-team/development_tools.md | 2 +- dev-docs/development_process.md | 2 +- dev-docs/hail-for-new-engineers.md | 4 +- dev-docs/ksync.md | 2 +- docker/hail-ubuntu/Dockerfile | 4 +- docker/hailgenetics/mirror_images.sh | 2 - docker/hailgenetics/python-dill/push.sh | 2 +- docker/third-party/images.txt | 3 - gear/pinned-requirements.txt | 52 ++- gear/requirements.txt | 6 + generate-linux-pip-lockfile.sh | 4 +- generate_pip_lockfile.sh | 4 +- hail/Dockerfile.hail-run | 2 +- hail/Makefile | 2 +- hail/python/dev/pinned-requirements.txt | 153 ++++--- hail/python/dev/requirements.txt | 6 +- hail/python/hail/docs/change_log.md | 13 +- hail/python/hail/docs/conf.py | 11 +- hail/python/hail/docs/index.rst | 2 +- hail/python/hail/docs/install/linux.rst | 6 +- hail/python/hail/docs/install/macosx.rst | 2 +- .../hail/docs/install/other-cluster.rst | 2 +- .../01-genome-wide-association-study.ipynb | 8 +- .../hail/docs/tutorials/08-plotting.ipynb | 4 +- hail/python/hail/experimental/plots.py | 24 +- hail/python/hail/expr/types.py | 2 +- hail/python/hail/plot/plots.py | 398 +++++++++++------- hail/python/hailtop/batch/backend.py | 4 +- hail/python/hailtop/batch/batch.py | 17 +- .../hailtop/batch/batch_pool_executor.py | 6 +- hail/python/hailtop/batch/docker.py | 6 +- hail/python/hailtop/batch/docs/change_log.rst | 12 + hail/python/hailtop/batch/docs/conf.py | 5 +- .../batch/docs/cookbook/random_forest.rst | 2 +- .../hailtop/batch/docs/getting_started.rst | 2 +- hail/python/hailtop/batch/docs/index.rst | 2 +- hail/python/hailtop/batch/job.py | 6 +- hail/python/hailtop/pinned-requirements.txt | 31 +- hail/python/pinned-requirements.txt | 227 +++++++--- hail/python/requirements.txt | 7 +- hail/python/setup.py | 4 +- hail/python/test/hail/expr/test_expr.py | 6 +- hail/python/test/hailtop/batch/test_batch.py | 2 +- .../hailtop/batch/test_batch_pool_executor.py | 5 +- memory/pinned-requirements.txt | 10 +- web_common/pinned-requirements.txt | 24 +- web_common/requirements.txt | 2 +- website/Makefile | 2 +- 57 files changed, 699 insertions(+), 479 deletions(-) diff --git a/Makefile b/Makefile index 8c954c09509..4845c54d497 100644 --- a/Makefile +++ b/Makefile @@ -75,13 +75,13 @@ install-dev-requirements: hail/python/hailtop/pinned-requirements.txt: hail/python/hailtop/requirements.txt ./generate-linux-pip-lockfile.sh hail/python/hailtop -hail/python/pinned-requirements.txt: hail/python/requirements.txt hail/python/hailtop/pinned-requirements.txt +hail/python/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt hail/python/requirements.txt ./generate-linux-pip-lockfile.sh hail/python -hail/python/dev/pinned-requirements.txt: hail/python/dev/requirements.txt hail/python/pinned-requirements.txt +hail/python/dev/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/requirements.txt ./generate-linux-pip-lockfile.sh hail/python/dev -gear/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt gear/requirements.txt +gear/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt hail/python/hailtop/pinned-requirements.txt gear/requirements.txt ./generate-linux-pip-lockfile.sh gear web_common/pinned-requirements.txt: gear/pinned-requirements.txt web_common/requirements.txt diff --git a/auth/pinned-requirements.txt b/auth/pinned-requirements.txt index 48082ef3ba5..188c0e1d477 100644 --- a/auth/pinned-requirements.txt +++ b/auth/pinned-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/auth/pinned-requirements.txt hail/auth/requirements.txt # @@ -22,7 +22,7 @@ charset-normalizer==3.1.0 # -c hail/auth/../hail/python/pinned-requirements.txt # -c hail/auth/../web_common/pinned-requirements.txt # requests -google-auth==2.17.2 +google-auth==2.17.3 # via # -c hail/auth/../gear/pinned-requirements.txt # -c hail/auth/../hail/python/pinned-requirements.txt @@ -38,13 +38,13 @@ idna==3.4 # requests oauthlib==3.2.2 # via requests-oauthlib -pyasn1==0.4.8 +pyasn1==0.5.0 # via # -c hail/auth/../gear/pinned-requirements.txt # -c hail/auth/../hail/python/pinned-requirements.txt # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via # -c hail/auth/../gear/pinned-requirements.txt # -c hail/auth/../hail/python/pinned-requirements.txt diff --git a/batch/Dockerfile.worker b/batch/Dockerfile.worker index 4b58071e37e..17e44e672de 100644 --- a/batch/Dockerfile.worker +++ b/batch/Dockerfile.worker @@ -8,7 +8,7 @@ RUN hail-apt-get-install \ xfsprogs \ libyajl-dev # crun runtime dependency -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 {% if global.cloud == "gcp" %} RUN echo "APT::Acquire::Retries \"5\";" > /etc/apt/apt.conf.d/80-retries && \ @@ -39,7 +39,7 @@ RUN hail-pip-install \ -r batch-requirements.txt \ pyspark==3.3.0 -ENV SPARK_HOME /usr/local/lib/python3.7/dist-packages/pyspark +ENV SPARK_HOME /usr/local/lib/python3.8/dist-packages/pyspark ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin" ENV PYSPARK_PYTHON python3 diff --git a/batch/pinned-requirements.txt b/batch/pinned-requirements.txt index 06e757cc048..6aa3a12599a 100644 --- a/batch/pinned-requirements.txt +++ b/batch/pinned-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/batch/pinned-requirements.txt hail/batch/requirements.txt # @@ -27,13 +27,7 @@ async-timeout==4.0.2 # -c hail/batch/../web_common/pinned-requirements.txt # -r hail/batch/requirements.txt # aiohttp -asynctest==0.13.0 - # via - # -c hail/batch/../gear/pinned-requirements.txt - # -c hail/batch/../hail/python/pinned-requirements.txt - # -c hail/batch/../web_common/pinned-requirements.txt - # aiohttp -attrs==22.2.0 +attrs==23.1.0 # via # -c hail/batch/../gear/pinned-requirements.txt # -c hail/batch/../hail/python/dev/pinned-requirements.txt @@ -70,17 +64,17 @@ multidict==6.0.4 # -c hail/batch/../web_common/pinned-requirements.txt # aiohttp # yarl -numpy==1.21.6 +numpy==1.24.3 # via # -c hail/batch/../hail/python/dev/pinned-requirements.txt # -c hail/batch/../hail/python/pinned-requirements.txt # pandas -packaging==23.0 +packaging==23.1 # via # -c hail/batch/../hail/python/dev/pinned-requirements.txt # -c hail/batch/../hail/python/pinned-requirements.txt # plotly -pandas==1.3.5 +pandas==2.0.1 # via # -c hail/batch/../hail/python/pinned-requirements.txt # -r hail/batch/requirements.txt @@ -111,15 +105,14 @@ tenacity==8.2.2 # plotly typing-extensions==4.5.0 # via - # -c hail/batch/../gear/pinned-requirements.txt # -c hail/batch/../hail/python/dev/pinned-requirements.txt # -c hail/batch/../hail/python/pinned-requirements.txt - # -c hail/batch/../web_common/pinned-requirements.txt # aiodocker - # aiohttp - # async-timeout - # yarl -yarl==1.8.2 +tzdata==2023.3 + # via + # -c hail/batch/../hail/python/pinned-requirements.txt + # pandas +yarl==1.9.1 # via # -c hail/batch/../gear/pinned-requirements.txt # -c hail/batch/../hail/python/pinned-requirements.txt diff --git a/batch/requirements.txt b/batch/requirements.txt index f7e98a4bc80..f6dc4b8397f 100644 --- a/batch/requirements.txt +++ b/batch/requirements.txt @@ -3,7 +3,7 @@ -c ../gear/pinned-requirements.txt -c ../web_common/pinned-requirements.txt dictdiffer>=0.8.1,<1 -pandas>=1.3.0,<1.5.0 +pandas>=2,<3 plotly>=5.5.0,<6 # Worker requirements aiodocker>=0.17.0,<1 diff --git a/benchmark/python/setup.py b/benchmark/python/setup.py index 807fbe85100..4b2f9c876e6 100755 --- a/benchmark/python/setup.py +++ b/benchmark/python/setup.py @@ -16,7 +16,7 @@ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=[ 'hail>=0.2', ], diff --git a/build.yaml b/build.yaml index ea1e0eff92c..3c5b835216a 100644 --- a/build.yaml +++ b/build.yaml @@ -154,7 +154,7 @@ steps: valueFrom: hail_ubuntu_image.image script: | set -ex - pip install pip-tools==6.9.0 + pip install pip-tools==6.13.0 cd /io/repo chmod 755 ./check_pip_requirements.sh ./check_pip_requirements.sh \ diff --git a/ci/Dockerfile.ci-utils b/ci/Dockerfile.ci-utils index aefd0f0b7f8..c8456c365d2 100644 --- a/ci/Dockerfile.ci-utils +++ b/ci/Dockerfile.ci-utils @@ -20,7 +20,7 @@ RUN hail-pip-install \ -r hailtop-requirements.txt \ -r gear-requirements.txt \ twine==1.11.0 \ - Jinja2==3.0.3 + 'Jinja2>3,<4' FROM golang:1.18 AS skopeo-build diff --git a/ci/pinned-requirements.txt b/ci/pinned-requirements.txt index f17d3427094..80aee4ab0c6 100644 --- a/ci/pinned-requirements.txt +++ b/ci/pinned-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/ci/pinned-requirements.txt hail/ci/requirements.txt # @@ -26,7 +26,7 @@ click==8.1.3 # via # -c hail/ci/../hail/python/dev/pinned-requirements.txt # zulip -cryptography==40.0.1 +cryptography==40.0.2 # via # -c hail/ci/../hail/python/pinned-requirements.txt # pyjwt @@ -41,10 +41,6 @@ idna==3.4 # -c hail/ci/../hail/python/pinned-requirements.txt # -c hail/ci/../web_common/pinned-requirements.txt # requests -importlib-metadata==3.10.1 - # via - # -c hail/ci/../hail/python/dev/pinned-requirements.txt - # click matrix-client==0.4.0 # via zulip pycparser==2.21 @@ -65,11 +61,8 @@ requests[security]==2.28.2 # zulip typing-extensions==4.5.0 # via - # -c hail/ci/../gear/pinned-requirements.txt # -c hail/ci/../hail/python/dev/pinned-requirements.txt # -c hail/ci/../hail/python/pinned-requirements.txt - # -c hail/ci/../web_common/pinned-requirements.txt - # importlib-metadata # zulip uritemplate==4.1.1 # via @@ -82,9 +75,5 @@ urllib3==1.26.15 # -c hail/ci/../hail/python/pinned-requirements.txt # matrix-client # requests -zipp==3.15.0 - # via - # -c hail/ci/../hail/python/dev/pinned-requirements.txt - # importlib-metadata zulip==0.8.2 # via -r hail/ci/requirements.txt diff --git a/dev-docs/compiler-team/development_tools.md b/dev-docs/compiler-team/development_tools.md index 8aeb9bb5d1d..1829031402d 100644 --- a/dev-docs/compiler-team/development_tools.md +++ b/dev-docs/compiler-team/development_tools.md @@ -51,7 +51,7 @@ https://www.anaconda.com/download/#macos After installing Anaconda, you should create a new dev environment for Hail with: - conda create --name hail python=3.7 + conda create --name hail python=3.8 and diff --git a/dev-docs/development_process.md b/dev-docs/development_process.md index ced29f6f48c..c730c72eff2 100644 --- a/dev-docs/development_process.md +++ b/dev-docs/development_process.md @@ -27,7 +27,7 @@ feature into smaller components. Before you can write code, there are some setup steps that will allow you to develop effectively. -Hail currently supports Python version 3.7 or greater. +Hail currently supports Python version 3.8 or greater. ``` make install-dev-requirements diff --git a/dev-docs/hail-for-new-engineers.md b/dev-docs/hail-for-new-engineers.md index c38a173d433..27be9ef69b6 100644 --- a/dev-docs/hail-for-new-engineers.md +++ b/dev-docs/hail-for-new-engineers.md @@ -154,8 +154,8 @@ We use a number of technologies: ### Services Technology -We almost exclusively write services in Python 3.7. We use a number of Python packages: -- [`asyncio`](https://docs.python.org/3.7/library/asyncio.html) for concurrency which is built on +We almost exclusively write services in Python 3.8. We use a number of Python packages: +- [`asyncio`](https://docs.python.org/3.8/library/asyncio.html) for concurrency which is built on [coroutines](https://en.wikipedia.org/wiki/Coroutine) not threads - [`aiohttp`](https://docs.aiohttp.org/en/stable/) for serving HTTPS requests (most services speak HTTPS) diff --git a/dev-docs/ksync.md b/dev-docs/ksync.md index 781d11ca118..55e1e3e8e3a 100644 --- a/dev-docs/ksync.md +++ b/dev-docs/ksync.md @@ -79,7 +79,7 @@ ksync watch 3. Create a spec in ~/.ksync/ksync.yaml using the create operation ``` -ksync create --local-read-only -l app=auth --name - -n jigold $(pwd)// /usr/local/lib/python3.7/dist-packages// +ksync create --local-read-only -l app=auth --name - -n jigold $(pwd)// /usr/local/lib/python3.8/dist-packages// ``` 4. Use ksync get to make sure the pods are being watched diff --git a/docker/hail-ubuntu/Dockerfile b/docker/hail-ubuntu/Dockerfile index a318e613b55..6835a621f0e 100644 --- a/docker/hail-ubuntu/Dockerfile +++ b/docker/hail-ubuntu/Dockerfile @@ -20,8 +20,8 @@ RUN chmod 755 /bin/retry && \ >> /etc/apt/sources.list && \ echo 'deb-src [signed-by=/usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \ >> /etc/apt/sources.list && \ - hail-apt-get-install python3.7-minimal python3.7-dev python3.7-distutils gcc g++ && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1 && \ + hail-apt-get-install python3.8-minimal python3.8-dev python3.8-distutils gcc g++ && \ + update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 && \ curl https://bootstrap.pypa.io/get-pip.py | python3 && \ python3 -m pip install 'pip>=21<22' && \ python3 -m pip check && \ diff --git a/docker/hailgenetics/mirror_images.sh b/docker/hailgenetics/mirror_images.sh index 3dfaf4354ca..7c001fe8331 100755 --- a/docker/hailgenetics/mirror_images.sh +++ b/docker/hailgenetics/mirror_images.sh @@ -32,8 +32,6 @@ then fi images=( - "python-dill:3.7" - "python-dill:3.7-slim" "python-dill:3.8" "python-dill:3.8-slim" "python-dill:3.9" diff --git a/docker/hailgenetics/python-dill/push.sh b/docker/hailgenetics/python-dill/push.sh index 97bee301732..4d79c97a322 100644 --- a/docker/hailgenetics/python-dill/push.sh +++ b/docker/hailgenetics/python-dill/push.sh @@ -2,7 +2,7 @@ set -ex -for version in 3.7 3.7-slim 3.8 3.8-slim 3.9 3.9-slim 3.10 3.10-slim +for version in 3.8 3.8-slim 3.9 3.9-slim 3.10 3.10-slim do public=hailgenetics/python-dill:$version diff --git a/docker/third-party/images.txt b/docker/third-party/images.txt index 3afb9550f7b..4e4dc2999bc 100644 --- a/docker/third-party/images.txt +++ b/docker/third-party/images.txt @@ -7,9 +7,6 @@ grafana/grafana:9.1.4 jupyter/scipy-notebook jupyter/scipy-notebook:c094bb7219f9 moby/buildkit:v0.8.3-rootless -python:3.7 -python:3.7-slim -python:3.7-slim-stretch python:3.8 python:3.8-slim python:3.9 diff --git a/gear/pinned-requirements.txt b/gear/pinned-requirements.txt index 323a40389ef..285d112ba82 100644 --- a/gear/pinned-requirements.txt +++ b/gear/pinned-requirements.txt @@ -1,11 +1,12 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/gear/pinned-requirements.txt hail/gear/requirements.txt # aiohttp==3.8.4 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp-session # kubernetes-asyncio @@ -15,50 +16,55 @@ aiomysql==0.1.1 # via -r hail/gear/requirements.txt aiosignal==1.3.1 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp async-timeout==4.0.2 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp -asynctest==0.13.0 - # via - # -c hail/gear/../hail/python/pinned-requirements.txt - # aiohttp -attrs==22.2.0 +attrs==23.1.0 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp cachetools==5.3.0 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-auth certifi==2022.12.7 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # kubernetes-asyncio # requests charset-normalizer==3.1.0 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp # requests frozenlist==1.3.3 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp # aiosignal google-api-core==2.11.0 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-api-python-client -google-api-python-client==2.84.0 +google-api-python-client==2.86.0 # via google-cloud-profiler -google-auth==2.17.2 +google-auth==2.17.3 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-api-core # google-api-python-client @@ -72,6 +78,7 @@ google-cloud-profiler==3.1.0 # via -r hail/gear/requirements.txt googleapis-common-protos==1.59.0 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-api-core httplib2==0.22.0 @@ -81,6 +88,7 @@ httplib2==0.22.0 idna==3.4 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # requests # yarl @@ -88,6 +96,7 @@ kubernetes-asyncio==19.15.1 # via -r hail/gear/requirements.txt multidict==6.0.4 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp # yarl @@ -100,16 +109,19 @@ prometheus-client==0.16.0 # prometheus-async protobuf==3.20.2 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-api-core # google-cloud-profiler -pyasn1==0.4.8 +pyasn1==0.5.0 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-auth pymysql==1.0.3 @@ -123,25 +135,30 @@ pyparsing==3.0.9 python-dateutil==2.8.2 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # kubernetes-asyncio pyyaml==6.0 # via + # -c hail/gear/../hail/python/dev/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # kubernetes-asyncio requests==2.28.2 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-api-core # google-cloud-profiler rsa==4.9 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-auth six==1.16.0 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # google-auth # google-auth-httplib2 @@ -149,21 +166,15 @@ six==1.16.0 # python-dateutil sortedcontainers==2.4.0 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # -r hail/gear/requirements.txt -typing-extensions==4.5.0 - # via - # -c hail/gear/../hail/python/dev/pinned-requirements.txt - # -c hail/gear/../hail/python/pinned-requirements.txt - # aiohttp - # aiohttp-session - # async-timeout - # yarl uritemplate==4.1.1 # via google-api-python-client urllib3==1.26.15 # via # -c hail/gear/../hail/python/dev/pinned-requirements.txt + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # kubernetes-asyncio # requests @@ -172,8 +183,9 @@ wrapt==1.15.0 # -c hail/gear/../hail/python/dev/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # prometheus-async -yarl==1.8.2 +yarl==1.9.1 # via + # -c hail/gear/../hail/python/hailtop/pinned-requirements.txt # -c hail/gear/../hail/python/pinned-requirements.txt # aiohttp diff --git a/gear/requirements.txt b/gear/requirements.txt index 6fba16e5c0a..2188d86531a 100644 --- a/gear/requirements.txt +++ b/gear/requirements.txt @@ -1,5 +1,11 @@ +# hailtop is installed in every service so we must be compatible with it +-c ../hail/python/hailtop/pinned-requirements.txt +# ci-utils includes gear and is used by test_dataproc which installs hail ergo we must be compatible +# with hail -c ../hail/python/pinned-requirements.txt +# dev is installed in the batch tests -c ../hail/python/dev/pinned-requirements.txt + aiohttp_session>=2.7,<2.13 aiomysql>=0.0.20,<1 google-cloud-profiler<4.0.0 diff --git a/generate-linux-pip-lockfile.sh b/generate-linux-pip-lockfile.sh index 34907fb5c03..a8636b297ba 100755 --- a/generate-linux-pip-lockfile.sh +++ b/generate-linux-pip-lockfile.sh @@ -9,8 +9,8 @@ PIP_COMPILE_IMAGE=hail-pip-compile:latest if [[ "$(docker images -q $PIP_COMPILE_IMAGE 2>/dev/null)" == "" ]]; then docker build -t $PIP_COMPILE_IMAGE -f - . <=2.2.1,<3 pytest-instafail>=0.4.2,<1 pytest-asyncio>=0.14.0,<1 pytest-timestamper>=0.0.9,<1 -sphinx>=3.5.4,<4 -sphinx-autodoc-typehints==1.11.1 +sphinx>=6,<7 +sphinx-autodoc-typehints==1.23.0 nbsphinx>=0.8.8,<1 sphinx_rtd_theme>=1.0.0,<2 jupyter>=1.0.0,<2 -# importlib-metadata<4: in dev/requirements.txt, jupyter depends on (an unpinned) ipykernel which needs importlib-metadata<4 -importlib-metadata<4 sphinxcontrib.katex>=0.9.0,<1 fswatch>=0.1.1,<1 matplotlib>=3.5,<4 diff --git a/hail/python/hail/docs/change_log.md b/hail/python/hail/docs/change_log.md index 2a7de77c311..63ab16e9485 100644 --- a/hail/python/hail/docs/change_log.md +++ b/hail/python/hail/docs/change_log.md @@ -1,4 +1,15 @@ -# Change Log +# Change Log And Version Policy + +## Python Version Compatibility Policy + +Hail complies with [NumPy's compatibility policy](https://numpy.org/neps/nep-0029-deprecation_policy.html#implementation) on Python +versions. In particular, Hail officially supports: + +- All minor versions of Python released 42 months prior to the project, and at minimum the two + latest minor versions. + +- All minor versions of numpy released in the 24 months prior to the project, and at minimum the + last three minor versions. ## Frequently Asked Questions diff --git a/hail/python/hail/docs/conf.py b/hail/python/hail/docs/conf.py index 2db95e2b844..fb181596164 100644 --- a/hail/python/hail/docs/conf.py +++ b/hail/python/hail/docs/conf.py @@ -48,7 +48,8 @@ # https://github.com/spatialaudio/nbsphinx/issues/24#issuecomment-187172022 and https://github.com/ContinuumIO/anaconda-issues/issues/1430 'IPython.sphinxext.ipython_console_highlighting', 'sphinx.ext.napoleon', - 'sphinx.ext.intersphinx' + 'sphinx.ext.intersphinx', + 'sphinx_rtd_theme', ] nbsphinx_timeout = 300 @@ -68,11 +69,11 @@ napoleon_use_param = False intersphinx_mapping = { - 'python': ('https://docs.python.org/3.7', None), + 'python': ('https://docs.python.org/3.8', None), 'PySpark': ('https://spark.apache.org/docs/latest/api/python/', None), - 'Bokeh': ('https://docs.bokeh.org/en/1.2.0/', None), + 'Bokeh': ('https://docs.bokeh.org/en/3.1.0/', None), 'numpy': ('https://numpy.org/doc/stable/', None), - 'scipy': ('https://docs.scipy.org/doc/scipy-1.3.3/reference', None), + 'scipy': ('https://docs.scipy.org/doc/scipy-1.9.3', None), 'pandas': ('https://pandas.pydata.org/docs/', None)} # Add any paths that contain templates here, relative to this directory. @@ -113,7 +114,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: diff --git a/hail/python/hail/docs/index.rst b/hail/python/hail/docs/index.rst index 615d42934f8..56c183236be 100644 --- a/hail/python/hail/docs/index.rst +++ b/hail/python/hail/docs/index.rst @@ -28,7 +28,7 @@ Contents Libraries For Software Developers Other Resources - Change Log + Change Log And Version Policy ================== Indices and tables diff --git a/hail/python/hail/docs/install/linux.rst b/hail/python/hail/docs/install/linux.rst index 5cb616a2cda..42221207edb 100644 --- a/hail/python/hail/docs/install/linux.rst +++ b/hail/python/hail/docs/install/linux.rst @@ -3,7 +3,7 @@ Install Hail on GNU/Linux ========================= - Install Java 8 or Java 11. -- Install Python 3.7 or later. +- Install Python 3.8 or later. - Install a recent version of the C and C++ standard libraries. GCC 5.0, LLVM version 3.4, or any later versions suffice. - Install BLAS and LAPACK. @@ -16,8 +16,8 @@ On a recent Debian-like system, the following should suffice: apt-get install -y \ openjdk-8-jre-headless \ g++ \ - python3.7 python3-pip \ + python3.8 python3-pip \ libopenblas-base liblapack3 - python3.7 -m pip install hail + python3.8 -m pip install hail `Now let's take Hail for a spin! `__ diff --git a/hail/python/hail/docs/install/macosx.rst b/hail/python/hail/docs/install/macosx.rst index 36244e6c169..c7797aeba88 100644 --- a/hail/python/hail/docs/install/macosx.rst +++ b/hail/python/hail/docs/install/macosx.rst @@ -11,6 +11,6 @@ Install Hail on Mac OS X brew tap homebrew/cask-versions brew install --cask temurin8 -- Install Python 3.7 or later. We recommend `Miniconda `__. +- Install Python 3.8 or later. We recommend `Miniconda `__. - Open Terminal.app and execute ``pip install hail``. - `Run your first Hail query! `__ diff --git a/hail/python/hail/docs/install/other-cluster.rst b/hail/python/hail/docs/install/other-cluster.rst index be022fd5947..cb57e656df8 100644 --- a/hail/python/hail/docs/install/other-cluster.rst +++ b/hail/python/hail/docs/install/other-cluster.rst @@ -11,7 +11,7 @@ Hail needs to be built from source on the leader node. Building Hail from source requires: - Java 8 or 11 JDK. -- Python 3.7 or later. +- Python 3.8 or later. - A recent C and a C++ compiler, GCC 5.0, LLVM 3.4, or later versions of either suffice. - The LZ4 library. diff --git a/hail/python/hail/docs/tutorials/01-genome-wide-association-study.ipynb b/hail/python/hail/docs/tutorials/01-genome-wide-association-study.ipynb index 9a67a1f2ee9..8bd520f758a 100644 --- a/hail/python/hail/docs/tutorials/01-genome-wide-association-study.ipynb +++ b/hail/python/hail/docs/tutorials/01-genome-wide-association-study.ipynb @@ -687,7 +687,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [], "source": [ "p = hl.plot.manhattan(gwas.p_value)\n", @@ -968,7 +970,7 @@ "metadata": { "anaconda-cloud": {}, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -982,7 +984,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.10.9" } }, "nbformat": 4, diff --git a/hail/python/hail/docs/tutorials/08-plotting.ipynb b/hail/python/hail/docs/tutorials/08-plotting.ipynb index bbee6a25abf..9a6335c1dbd 100644 --- a/hail/python/hail/docs/tutorials/08-plotting.ipynb +++ b/hail/python/hail/docs/tutorials/08-plotting.ipynb @@ -168,7 +168,7 @@ " label=common_mt.cols()[pca_scores.s].SuperPopulation,\n", " title='PCA (downsampled)', xlabel='PC1', ylabel='PC2', collect_all=False, n_divisions=50)\n", "\n", - "show(gridplot([p, p2], ncols=2, plot_width=400, plot_height=400))" + "show(gridplot([p, p2], ncols=2, width=400, height=400))" ] }, { @@ -208,7 +208,7 @@ "p = hl.plot.qq(gwas.p_value, collect_all=True)\n", "p2 = hl.plot.qq(gwas.p_value, n_divisions=75)\n", "\n", - "show(gridplot([p, p2], ncols=2, plot_width=400, plot_height=400))" + "show(gridplot([p, p2], ncols=2, width=400, height=400))" ] }, { diff --git a/hail/python/hail/experimental/plots.py b/hail/python/hail/experimental/plots.py index 8f0a8537d10..ad8369a0605 100644 --- a/hail/python/hail/experimental/plots.py +++ b/hail/python/hail/experimental/plots.py @@ -4,7 +4,7 @@ import hail as hl from bokeh.layouts import gridplot -from bokeh.models import Title, ColumnDataSource, HoverTool, Div, Tabs, Panel +from bokeh.models import Title, ColumnDataSource, HoverTool, Div, Tabs, TabPanel from bokeh.palettes import Spectral8 from bokeh.plotting import figure from bokeh.transform import factor_cmap @@ -39,7 +39,7 @@ def plot_roc_curve(ht, scores, tp_label='tp', fp_label='fp', colors=None, title= Returns ------- - :obj:`tuple` of :class:`bokeh.plotting.figure.Figure` and :obj:`list` of :class:`str` + :obj:`tuple` of :class:`bokeh.plotting.figure` and :obj:`list` of :class:`str` Figure, and list of AUCs corresponding to scores. """ if colors is None: @@ -71,7 +71,7 @@ def plot_roc_curve(ht, scores, tp_label='tp', fp_label='fp', colors=None, title= auc = ordered_ht.aggregate(hl.agg.sum(ordered_ht.auc_contrib)) aucs.append(auc) df = ordered_ht.annotate(score_name=ordered_ht.score_name + f' (AUC = {auc:.4f})').to_pandas() - p.line(x='fpr', y='tpr', legend='score_name', source=ColumnDataSource(df), color=colors[score], line_width=3) + p.line(x='fpr', y='tpr', legend_field='score_name', source=ColumnDataSource(df), color=colors[score], line_width=3) p.legend.location = 'bottom_right' p.legend.click_policy = 'hide' @@ -91,7 +91,7 @@ def hail_metadata(t_path): Returns ------- - :class:`bokeh.plotting.figure.Figure` or :class:`bokeh.models.layouts.Column` + :class:`bokeh.plotting.figure` or :class:`bokeh.models.layouts.Column` """ def get_rows_data(rows_files): file_sizes = [] @@ -173,7 +173,7 @@ def scale_file_sizes(file_sizes): if not row_partition_bounds: warning('Table is not partitioned. Only plotting file sizes') row_file_sizes_hist, row_file_sizes_edges = np.histogram(row_file_sizes, bins=50) - p_file_size = figure(plot_width=panel_size, plot_height=panel_size) + p_file_size = figure(width=panel_size, height=panel_size) p_file_size.quad(right=row_file_sizes_hist, left=0, bottom=row_file_sizes_edges[:-1], top=row_file_sizes_edges[1:], fill_color="#036564", line_color="#033649") p_file_size.yaxis.axis_label = f'File size ({row_scale}B)' @@ -208,7 +208,7 @@ def scale_file_sizes(file_sizes): tools = "hover,save,pan,box_zoom,reset,wheel_zoom" source = ColumnDataSource(pd.DataFrame(all_data)) - p = figure(tools=tools, plot_width=panel_size, plot_height=panel_size) + p = figure(tools=tools, width=panel_size, height=panel_size) p.title.text = title p.xaxis.axis_label = 'Number of rows' p.yaxis.axis_label = f'File size ({row_scale}B)' @@ -220,8 +220,8 @@ def scale_file_sizes(file_sizes): ('rows_per_partition', 'row_file_sizes_human', 'partition_bounds', 'index')] p_stats = Div(text=msg) - p_rows_per_partition = figure(x_range=p.x_range, plot_width=panel_size, plot_height=subpanel_size) - p_file_size = figure(y_range=p.y_range, plot_width=subpanel_size, plot_height=panel_size) + p_rows_per_partition = figure(x_range=p.x_range, width=panel_size, height=subpanel_size) + p_file_size = figure(y_range=p.y_range, width=subpanel_size, height=panel_size) rows_per_partition_hist, rows_per_partition_edges = np.histogram(all_data['rows_per_partition'], bins=50) p_rows_per_partition.quad(top=rows_per_partition_hist, bottom=0, left=rows_per_partition_edges[:-1], @@ -241,7 +241,7 @@ def scale_file_sizes(file_sizes): msg += success_file[0] source = ColumnDataSource(pd.DataFrame(all_data)) - p = figure(tools=tools, plot_width=panel_size, plot_height=panel_size) + p = figure(tools=tools, width=panel_size, height=panel_size) p.title.text = title p.xaxis.axis_label = 'Number of rows' p.yaxis.axis_label = f'File size ({entry_scale}B)' @@ -251,17 +251,17 @@ def scale_file_sizes(file_sizes): p.select_one(HoverTool).tooltips = [(x, f'@{x}') for x in ('rows_per_partition', 'entry_file_sizes_human', 'partition_bounds', 'index')] p_stats = Div(text=msg) - p_rows_per_partition = figure(x_range=p.x_range, plot_width=panel_size, plot_height=subpanel_size) + p_rows_per_partition = figure(x_range=p.x_range, width=panel_size, height=subpanel_size) p_rows_per_partition.quad(top=rows_per_partition_hist, bottom=0, left=rows_per_partition_edges[:-1], right=rows_per_partition_edges[1:], fill_color="#036564", line_color="#033649") - p_file_size = figure(y_range=p.y_range, plot_width=subpanel_size, plot_height=panel_size) + p_file_size = figure(y_range=p.y_range, width=subpanel_size, height=panel_size) row_file_sizes_hist, row_file_sizes_edges = np.histogram(all_data['entry_file_sizes'], bins=50) p_file_size.quad(right=row_file_sizes_hist, left=0, bottom=row_file_sizes_edges[:-1], top=row_file_sizes_edges[1:], fill_color="#036564", line_color="#033649") entries_grid = gridplot([[p_rows_per_partition, p_stats], [p, p_file_size]]) - return Tabs(tabs=[Panel(child=entries_grid, title='Entries'), Panel(child=rows_grid, title='Rows')]) + return Tabs(tabs=[TabPanel(child=entries_grid, title='Entries'), TabPanel(child=rows_grid, title='Rows')]) else: return rows_grid diff --git a/hail/python/hail/expr/types.py b/hail/python/hail/expr/types.py index b6ef0b259d2..d7721349c3d 100644 --- a/hail/python/hail/expr/types.py +++ b/hail/python/hail/expr/types.py @@ -2129,7 +2129,7 @@ def from_numpy(np_dtype): return tfloat32 elif np_dtype == np.float64: return tfloat64 - elif np_dtype == np.bool: + elif np_dtype == np.bool_: return tbool else: raise ValueError(f"numpy type {np_dtype} could not be converted to a hail type.") diff --git a/hail/python/hail/plot/plots.py b/hail/python/hail/plot/plots.py index f1395aa3220..92bba9eb7c2 100644 --- a/hail/python/hail/plot/plots.py +++ b/hail/python/hail/plot/plots.py @@ -5,10 +5,14 @@ import pandas as pd import bokeh import bokeh.io -from bokeh.models import HoverTool, ColorBar, LogTicker, LogColorMapper, LinearColorMapper, CategoricalColorMapper, \ - ColumnDataSource, BasicTicker, Plot, ColorMapper, CDSView, GroupFilter, Legend, LegendItem, Renderer, CustomJS, \ - Select, Column, Span, DataRange1d, Slope, Label -from bokeh.plotting import figure, Figure +import bokeh.models +from bokeh.models import (HoverTool, ColorBar, LogTicker, LogColorMapper, LinearColorMapper, + CategoricalColorMapper, ColumnDataSource, BasicTicker, Plot, CDSView, + GroupFilter, IntersectionFilter, Legend, LegendItem, Renderer, CustomJS, + Select, Column, Span, DataRange1d, Slope, Label, ColorMapper, GridPlot) +import bokeh.plotting +import bokeh.palettes +from bokeh.plotting import figure from bokeh.transform import transform from bokeh.layouts import gridplot @@ -20,10 +24,10 @@ check_row_indexed from hail.typecheck import typecheck, oneof, nullable, sized_tupleof, numeric, \ sequenceof, dictof -from hail import Table +from hail import Table, MatrixTable from hail.utils.struct import Struct from hail.utils.java import warning -from typing import List, Tuple, Dict, Union, Callable +from typing import List, Tuple, Dict, Union, Callable, Optional, Sequence, Any, Set import hail palette = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf'] @@ -56,7 +60,7 @@ def show(obj, interact=None): interact(handle) -def cdf(data, k=350, legend=None, title=None, normalize=True, log=False) -> Figure: +def cdf(data, k=350, legend=None, title=None, normalize=True, log=False) -> figure: """Create a cumulative density plot. Parameters @@ -76,11 +80,11 @@ def cdf(data, k=350, legend=None, title=None, normalize=True, log=False) -> Figu Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.plotting.figure` """ if isinstance(data, Expression): if data._indices is None: - return ValueError('Invalid input') + raise ValueError('Invalid input') agg_f = data._aggregation_method() data = agg_f(aggregators.approx_cdf(data, k)) @@ -100,8 +104,8 @@ def cdf(data, k=350, legend=None, title=None, normalize=True, log=False) -> Figu x_axis_label=legend, y_axis_label=y_axis_label, y_axis_type=y_axis_type, - plot_width=600, - plot_height=400, + width=600, + height=400, background_fill_color='#EEEEEE', tools='xpan,xwheel_zoom,reset,save', active_scroll='xwheel_zoom') @@ -155,10 +159,10 @@ def update_grid_size(p): return 1 / p + math.sqrt(math.log(2 * p / failure_prob) * s / 2) -def pdf(data, k=1000, confidence=5, legend=None, title=None, log=False, interactive=False) -> Union[Figure, Tuple[Figure, Callable]]: +def pdf(data, k=1000, confidence=5, legend=None, title=None, log=False, interactive=False) -> Union[figure, Tuple[figure, Callable]]: if isinstance(data, Expression): if data._indices is None: - return ValueError('Invalid input') + raise ValueError('Invalid input') agg_f = data._aggregation_method() data = agg_f(aggregators.approx_cdf(data, k)) @@ -175,8 +179,8 @@ def pdf(data, k=1000, confidence=5, legend=None, title=None, log=False, interact x_axis_label=legend, y_axis_label=y_axis_label, y_axis_type=y_axis_type, - plot_width=600, - plot_height=400, + width=600, + height=400, tools='xpan,xwheel_zoom,reset,save', active_scroll='xwheel_zoom', background_fill_color='#EEEEEE') @@ -205,7 +209,7 @@ def update(confidence=confidence): else: new_data = {'left': [min_x, *x[keep]], 'right': [*x[keep], max_x], 'bottom': np.full(len(slopes), 0), 'top': slopes} plot.data_source.data = new_data - bokeh.io.push_notebook(handle) + bokeh.io.push_notebook(handle=handle) from ipywidgets import interact interact(update, confidence=(1, 10, .01)) @@ -291,7 +295,7 @@ def compare(x1, y1, x2, y2): return new_y, keep -def smoothed_pdf(data, k=350, smoothing=.5, legend=None, title=None, log=False, interactive=False, figure=None) -> Union[Figure, Tuple[Figure, Callable]]: +def smoothed_pdf(data, k=350, smoothing=.5, legend=None, title=None, log=False, interactive=False, figure=None) -> Union[figure, Tuple[figure, Callable]]: """Create a density plot. Parameters @@ -310,16 +314,16 @@ def smoothed_pdf(data, k=350, smoothing=.5, legend=None, title=None, log=False, Plot the log10 of the bin counts. interactive : bool If `True`, return a handle to pass to :func:`bokeh.io.show`. - figure : :class:`bokeh.plotting.figure.Figure` + figure : :class:`bokeh.plotting.figure` If not None, add density plot to figure. Otherwise, create a new figure. Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.plotting.figure` """ if isinstance(data, Expression): if data._indices is None: - return ValueError('Invalid input') + raise ValueError('Invalid input') agg_f = data._aggregation_method() data = agg_f(aggregators.approx_cdf(data, k)) @@ -338,8 +342,8 @@ def smoothed_pdf(data, k=350, smoothing=.5, legend=None, title=None, log=False, x_axis_label=legend, y_axis_label=y_axis_label, y_axis_type=y_axis_type, - plot_width=600, - plot_height=400, + width=600, + height=400, tools='xpan,xwheel_zoom,reset,save', active_scroll='xwheel_zoom', background_fill_color='#EEEEEE') @@ -370,7 +374,7 @@ def mk_interact(handle): def update(smoothing=smoothing): final = f(x_d, round1, smoothing) line.data_source.data = {'x': x_d, 'y': final} - bokeh.io.push_notebook(handle) + bokeh.io.push_notebook(handle=handle) from ipywidgets import interact interact(update, smoothing=(.02, .8, .005)) @@ -382,7 +386,7 @@ def update(smoothing=smoothing): @typecheck(data=oneof(Struct, expr_float64), range=nullable(sized_tupleof(numeric, numeric)), bins=int, legend=nullable(str), title=nullable(str), log=bool, interactive=bool) -def histogram(data, range=None, bins=50, legend=None, title=None, log=False, interactive=False) -> Union[Figure, Tuple[Figure, Callable]]: +def histogram(data, range=None, bins=50, legend=None, title=None, log=False, interactive=False) -> Union[figure, Tuple[figure, Callable]]: """Create a histogram. Notes @@ -407,7 +411,7 @@ def histogram(data, range=None, bins=50, legend=None, title=None, log=False, int Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.plotting.figure` """ if isinstance(data, Expression): if data._indices.source is not None: @@ -425,7 +429,7 @@ def histogram(data, range=None, bins=50, legend=None, title=None, log=False, int raise ValueError("'data' contains no values that are defined and finite") data = agg_f(aggregators.hist(data, start, end, bins)) else: - return ValueError('Invalid input') + raise ValueError('Invalid input') elif 'values' in data: cdf = data hist, edges = np.histogram(cdf['values'], bins=bins, weights=np.diff(cdf.ranks), density=True) @@ -492,7 +496,7 @@ def update(bins=bins, phase=0): hist, edges = np.histogram(cdf['values'], bins=edges, weights=np.diff(cdf.ranks), density=True) new_data = {'top': hist, 'left': edges[:-1], 'right': edges[1:], 'bottom': np.full(len(hist), 0)} q.data_source.data = new_data - bokeh.io.push_notebook(handle) + bokeh.io.push_notebook(handle=handle) from ipywidgets import interact interact(update, bins=(0, 5 * bins), phase=(0, 1, .01)) @@ -504,7 +508,7 @@ def update(bins=bins, phase=0): @typecheck(data=oneof(Struct, expr_float64), range=nullable(sized_tupleof(numeric, numeric)), bins=int, legend=nullable(str), title=nullable(str), normalize=bool, log=bool) -def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, normalize=True, log=False) -> Figure: +def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, normalize=True, log=False) -> figure: """Create a cumulative histogram. Parameters @@ -526,7 +530,7 @@ def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, nor Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.plotting.figure` """ if isinstance(data, Expression): if data._indices.source is not None: @@ -538,7 +542,7 @@ def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, nor start, end = agg_f((aggregators.min(data), aggregators.max(data))) data = agg_f(aggregators.hist(data, start, end, bins)) else: - return ValueError('Invalid input') + raise ValueError('Invalid input') if legend is None: legend = "" @@ -560,20 +564,20 @@ def cumulative_histogram(data, range=None, bins=50, legend=None, title=None, nor return p -@typecheck(p=bokeh.plotting.Figure, font_size=str) +@typecheck(p=figure, font_size=str) def set_font_size(p, font_size: str = '12pt'): """Set most of the font sizes in a bokeh figure Parameters ---------- - p : :class:`bokeh.plotting.figure.Figure` + p : :class:`bokeh.plotting.figure` Input figure. font_size : str String of font size in points (e.g. '12pt'). Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.plotting.figure` """ p.legend.label_text_font_size = font_size p.xaxis.axis_label_text_font_size = font_size @@ -596,12 +600,12 @@ def set_font_size(p, font_size: str = '12pt'): def histogram2d(x: NumericExpression, y: NumericExpression, bins: int = 40, - range: Tuple[int, int] = None, - title: str = None, + range: Optional[Tuple[int, int]] = None, + title: Optional[str] = None, width: int = 600, height: int = 600, - colors: List[str] = bokeh.palettes.all_palettes['Blues'][7][::-1], - log: bool = False) -> Figure: + colors: Sequence[str] = bokeh.palettes.all_palettes['Blues'][7][::-1], + log: bool = False) -> figure: """Plot a two-dimensional histogram. ``x`` and ``y`` must both be a :class:`.NumericExpression` from the same :class:`.Table`. @@ -640,7 +644,7 @@ def histogram2d(x: NumericExpression, Plot height (default 600px). title : str Title of the plot. - colors : List[str] + colors : Sequence[str] List of colors (hex codes, or strings as described `here `__). Compatible with one of the many built-in palettes available `here `__. @@ -649,7 +653,7 @@ def histogram2d(x: NumericExpression, Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.plotting.figure` """ data = _generate_hist2d_data(x, y, bins, range).to_pandas() @@ -657,6 +661,7 @@ def histogram2d(x: NumericExpression, data['x'] = data['x'].apply(lambda e: str(float(e))) data['y'] = data['y'].apply(lambda e: str(float(e))) + mapper: ColorMapper if log: mapper = LogColorMapper(palette=colors, low=data.c.min(), high=data.c.max()) else: @@ -666,7 +671,7 @@ def histogram2d(x: NumericExpression, y_axis = sorted(set(data.y), key=lambda z: float(z)) p = figure(title=title, x_range=x_axis, y_range=y_axis, - x_axis_location="above", plot_width=width, plot_height=height, + x_axis_location="above", width=width, height=height, tools="hover,save,pan,box_zoom,reset,wheel_zoom", toolbar_location='below') p.grid.grid_line_color = None @@ -686,7 +691,10 @@ def histogram2d(x: NumericExpression, label_standoff=12 if log else 6, border_line_color=None, location=(0, 0)) p.add_layout(color_bar, 'right') - p.select_one(HoverTool).tooltips = [('x', '@x'), ('y', '@y',), ('count', '@c')] + hovertool = p.select_one(HoverTool) + assert hovertool is not None + hovertool.tooltips = [('x', '@x'), ('y', '@y',), ('count', '@c')] + return p @@ -746,8 +754,8 @@ def frange(start, stop, step): def _collect_scatter_plot_data( x: Tuple[str, NumericExpression], y: Tuple[str, NumericExpression], - fields: Dict[str, Expression] = None, - n_divisions: int = None, + fields: Optional[Dict[str, Expression]] = None, + n_divisions: Optional[int] = None, missing_label: str = 'NA' ) -> pd.DataFrame: @@ -783,8 +791,9 @@ def _collect_scatter_plot_data( return source_pd -def _get_categorical_palette(factors: List[str]) -> Dict[str, str]: +def _get_categorical_palette(factors: List[str]) -> ColorMapper: n = max(3, len(factors)) + _palette: Sequence[str] if n < len(palette): _palette = palette elif n < 21: @@ -798,23 +807,43 @@ def _get_categorical_palette(factors: List[str]) -> Dict[str, str]: def _get_scatter_plot_elements( - sp: Plot, source_pd: pd.DataFrame, x_col: str, y_col: str, label_cols: List[str], - colors: Dict[str, ColorMapper] = None, size: int = 4, -) -> Tuple[bokeh.plotting.Figure, Dict[str, List[LegendItem]], Legend, ColorBar, Dict[str, ColorMapper], List[Renderer]]: + sp: Plot, + source_pd: pd.DataFrame, + x_col: str, + y_col: str, + label_cols: List[str], + colors: Optional[Dict[str, ColorMapper]] = None, + size: int = 4, + hover_cols: Optional[Set[str]] = None, +) -> Union[Tuple[Plot, Dict[str, List[LegendItem]], Legend, ColorBar, Dict[str, ColorMapper], List[Renderer]], + Tuple[Plot, None, None, None, None, None]]: if not source_pd.shape[0]: print("WARN: No data to plot.") return sp, None, None, None, None, None - sp.tools.append(HoverTool(tooltips=[(x_col, f'@{x_col}'), (y_col, f'@{y_col}')] - + [(c, f'@{c}') for c in source_pd.columns if c not in [x_col, y_col]])) + possible_tooltips = [ + (x_col, f'@{x_col}'), + (y_col, f'@{y_col}') + ] + [ + (c, f'@{c}') + for c in source_pd.columns + if c not in [x_col, y_col] + ] + + if hover_cols is not None: + possible_tooltips = [ + x + for x in possible_tooltips + if x[0] in hover_cols + ] + sp.tools.append(HoverTool(tooltips=possible_tooltips)) cds = ColumnDataSource(source_pd) if not label_cols: sp.circle(x_col, y_col, source=cds, size=size) return sp, None, None, None, None, None - continuous_cols = [col for col in label_cols if (str(source_pd.dtypes[col]).startswith('float') or str(source_pd.dtypes[col]).startswith('int'))] @@ -823,7 +852,7 @@ def _get_scatter_plot_elements( # Assign color mappers to columns if colors is None: colors = {} - color_mappers = {} + color_mappers: Dict[str, ColorMapper] = {} for col in continuous_cols: low = np.nanmin(source_pd[col]) @@ -844,7 +873,7 @@ def _get_scatter_plot_elements( # Create initial glyphs initial_col = label_cols[0] initial_mapper = color_mappers[initial_col] - legend_items = {} + legend_items: Dict[str, List[LegendItem]] = {} if not factor_cols: all_renderers = [ @@ -853,20 +882,26 @@ def _get_scatter_plot_elements( else: all_renderers = [] - legend_items = {col: collections.defaultdict(list) for col in factor_cols} + legend_items_by_key_by_factor = {col: collections.defaultdict(list) for col in factor_cols} for key in source_pd.groupby(factor_cols).groups.keys(): key = key if len(factor_cols) > 1 else [key] - cds_view = CDSView(source=cds, filters=[GroupFilter(column_name=factor_cols[i], group=key[i]) for i in range(0, len(factor_cols))]) + cds_view = CDSView( + filter=IntersectionFilter(operands=[GroupFilter(column_name=factor_cols[i], group=key[i]) for i in range(0, len(factor_cols))]) + ) renderer = sp.circle(x_col, y_col, color=transform(initial_col, initial_mapper), source=cds, view=cds_view, size=size) all_renderers.append(renderer) for i in range(0, len(factor_cols)): - legend_items[factor_cols[i]][key[i]].append(renderer) + legend_items_by_key_by_factor[factor_cols[i]][key[i]].append(renderer) - legend_items = {factor: [LegendItem(label=key, renderers=renderers) for key, renderers in key_renderers.items()] for factor, key_renderers in legend_items.items()} + legend_items = {factor: [LegendItem(label=key, renderers=renderers) + for key, renderers in key_renderers.items()] + for factor, key_renderers in legend_items_by_key_by_factor.items()} # Add legend / color bar legend = Legend(visible=False, click_policy='hide', orientation='vertical') if initial_col not in factor_cols else Legend(items=legend_items[initial_col], click_policy='hide', orientation='vertical') - color_bar = ColorBar(visible=False) if initial_col not in continuous_cols else ColorBar(color_mapper=color_mappers[initial_col]) + color_bar = ColorBar(color_mapper=color_mappers[initial_col]) + if initial_col not in continuous_cols: + color_bar.visible = False sp.add_layout(legend, 'left') sp.add_layout(color_bar, 'left') @@ -883,30 +918,30 @@ def _get_scatter_plot_elements( def scatter( x: Union[NumericExpression, Tuple[str, NumericExpression]], y: Union[NumericExpression, Tuple[str, NumericExpression]], - label: Union[Expression, Dict[str, Expression]] = None, - title: str = None, - xlabel: str = None, - ylabel: str = None, + label: Optional[Union[Expression, Dict[str, Expression]]] = None, + title: Optional[str] = None, + xlabel: Optional[str] = None, + ylabel: Optional[str] = None, size: int = 4, legend: bool = True, - hover_fields: Dict[str, Expression] = None, - colors: Union[ColorMapper, Dict[str, ColorMapper]] = None, + hover_fields: Optional[Dict[str, Expression]] = None, + colors: Optional[Union[ColorMapper, Dict[str, ColorMapper]]] = None, width: int = 800, height: int = 800, collect_all: bool = False, - n_divisions: int = 500, + n_divisions: Optional[int] = 500, missing_label: str = 'NA' -) -> Union[Figure, Column]: +) -> Union[Plot, Column]: """Create an interactive scatter plot. ``x`` and ``y`` must both be either: - a :class:`.NumericExpression` from the same :class:`.Table`. - a tuple (str, :class:`.NumericExpression`) from the same :class:`.Table`. If passed as a tuple the first element is used as the hover label. - If no label or a single label is provided, then returns :class:`bokeh.plotting.figure.Figure` + If no label or a single label is provided, then returns :class:`bokeh.plotting.figure` Otherwise returns a :class:`bokeh.models.layouts.Column` containing: - a :class:`bokeh.models.widgets.inputs.Select` dropdown selection widget for labels - - a :class:`bokeh.plotting.figure.Figure` containing the interactive scatter plot + - a :class:`bokeh.plotting.figure` containing the interactive scatter plot Points will be colored by one of the labels defined in the ``label`` using the color scheme defined in the corresponding entry of ``colors`` if provided (otherwise a default scheme is used). To specify your color @@ -925,25 +960,25 @@ def scatter( List of x-values to be plotted. y : :class:`.NumericExpression` or (str, :class:`.NumericExpression`) List of y-values to be plotted. - label : :class:`.Expression` or Dict[str, :class:`.Expression`]] + label : :class:`.Expression` or Dict[str, :class:`.Expression`]], optional Either a single expression (if a single label is desired), or a dictionary of label name -> label value for x and y values. Used to color each point w.r.t its label. When multiple labels are given, a dropdown will be displayed with the different options. Can be used with categorical or continuous expressions. - title : str + title : str, optional Title of the scatterplot. - xlabel : str + xlabel : str, optional X-axis label. - ylabel : str + ylabel : str, optional Y-axis label. size : int Size of markers in screen space units. legend: bool Whether or not to show the legend in the resulting figure. - hover_fields : Dict[str, :class:`.Expression`] + hover_fields : Dict[str, :class:`.Expression`], optional Extra fields to be displayed when hovering over a point on the plot. - colors : :class:`bokeh.models.mappers.ColorMapper` or Dict[str, :class:`bokeh.models.mappers.ColorMapper`] + colors : :class:`bokeh.models.mappers.ColorMapper` or Dict[str, :class:`bokeh.models.mappers.ColorMapper`], optional If a single label is used, then this can be a color mapper, if multiple labels are used, then this should be a Dict of label name -> color mapper. Used to set colors for the labels defined using ``label``. @@ -954,35 +989,62 @@ def scatter( Plot height collect_all : bool Whether to collect all values or downsample before plotting. - n_divisions : int + n_divisions : int, optional Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints. missing_label: str Label to use when a point is missing data for a categorical label Returns ------- - :class:`bokeh.plotting.figure.Figure` if no label or a single label was given, otherwise :class:`bokeh.models.layouts.Column` + :class:`bokeh.models.Plot` if no label or a single label was given, otherwise :class:`bokeh.models.layouts.Column` """ hover_fields = {} if hover_fields is None else hover_fields - label = {} if label is None else {'label': label} if isinstance(label, Expression) else label - colors = {'label': colors} if isinstance(colors, ColorMapper) else colors - label_cols = list(label.keys()) + + label_by_col: Dict[str, Expression] + if label is None: + label_by_col = {} + elif isinstance(label, Expression): + label_by_col = {'label': label} + else: + assert isinstance(label, dict) + label_by_col = label + + if isinstance(colors, ColorMapper): + colors_by_col = {'label': colors} + else: + colors_by_col = colors + + label_cols = list(label_by_col.keys()) if isinstance(x, NumericExpression): - x = ('x', x) + _x = ('x', x) + else: + _x = x if isinstance(y, NumericExpression): - y = ('y', y) + _y = ('y', y) + else: + _y = y - source_pd = _collect_scatter_plot_data(x, y, fields={**hover_fields, **label}, n_divisions=None if collect_all else n_divisions, missing_label=missing_label) + source_pd = _collect_scatter_plot_data(_x, + _y, + fields={**hover_fields, **label_by_col}, + n_divisions=None if collect_all else n_divisions, + missing_label=missing_label) sp = figure(title=title, x_axis_label=xlabel, y_axis_label=ylabel, height=height, width=width) - sp, sp_legend_items, sp_legend, sp_color_bar, sp_color_mappers, sp_scatter_renderers = _get_scatter_plot_elements(sp, source_pd, x[0], y[0], label_cols, colors, size) + sp, sp_legend_items, sp_legend, sp_color_bar, sp_color_mappers, sp_scatter_renderers = _get_scatter_plot_elements( + sp, source_pd, _x[0], _y[0], label_cols, colors_by_col, size, + hover_cols={'x', 'y'} | set(hover_fields) + ) if not legend: + assert sp_legend is not None + assert sp_color_bar is not None sp_legend.visible = False sp_color_bar.visible = False # If multiple labels, create JS call back selector if len(label_cols) > 1: + callback_args: Dict[str, Any] callback_args = dict( color_mappers=sp_color_mappers, scatter_renderers=sp_scatter_renderers @@ -1032,27 +1094,27 @@ def scatter( def joint_plot( x: Union[NumericExpression, Tuple[str, NumericExpression]], y: Union[NumericExpression, Tuple[str, NumericExpression]], - label: Union[Expression, Dict[str, Expression]] = None, - title: str = None, - xlabel: str = None, - ylabel: str = None, + label: Optional[Union[Expression, Dict[str, Expression]]] = None, + title: Optional[str] = None, + xlabel: Optional[str] = None, + ylabel: Optional[str] = None, size: int = 4, legend: bool = True, - hover_fields: Dict[str, StringExpression] = None, - colors: Union[ColorMapper, Dict[str, ColorMapper]] = None, + hover_fields: Optional[Dict[str, StringExpression]] = None, + colors: Optional[Union[ColorMapper, Dict[str, ColorMapper]]] = None, width: int = 800, height: int = 800, collect_all: bool = False, - n_divisions: int = 500, + n_divisions: Optional[int] = 500, missing_label: str = 'NA' -) -> Column: +) -> GridPlot: """Create an interactive scatter plot with marginal densities on the side. ``x`` and ``y`` must both be either: - a :class:`.NumericExpression` from the same :class:`.Table`. - a tuple (str, :class:`.NumericExpression`) from the same :class:`.Table`. If passed as a tuple the first element is used as the hover label. - This function returns a :class:`bokeh.models.layouts.Column` containing two :class:`bokeh.plotting.figure.Row`: + This function returns a :class:`bokeh.models.layouts.Column` containing two :class:`figure.Row`: - The first row contains the X-axis marginal density and a selection widget if multiple entries are specified in the ``label`` - The second row contains the scatter plot and the y-axis marginal density @@ -1074,25 +1136,25 @@ def joint_plot( List of x-values to be plotted. y : :class:`.NumericExpression` or (str, :class:`.NumericExpression`) List of y-values to be plotted. - label : :class:`.Expression` or Dict[str, :class:`.Expression`]] + label : :class:`.Expression` or Dict[str, :class:`.Expression`]], optional Either a single expression (if a single label is desired), or a dictionary of label name -> label value for x and y values. Used to color each point w.r.t its label. When multiple labels are given, a dropdown will be displayed with the different options. Can be used with categorical or continuous expressions. - title : str + title : str, optional Title of the scatterplot. - xlabel : str + xlabel : str, optional X-axis label. - ylabel : str + ylabel : str, optional Y-axis label. size : int Size of markers in screen space units. legend: bool Whether or not to show the legend in the resulting figure. - hover_fields : Dict[str, :class:`.Expression`] + hover_fields : Dict[str, :class:`.Expression`], optional Extra fields to be displayed when hovering over a point on the plot. - colors : :class:`bokeh.models.mappers.ColorMapper` or Dict[str, :class:`bokeh.models.mappers.ColorMapper`] + colors : :class:`bokeh.models.mappers.ColorMapper` or Dict[str, :class:`bokeh.models.mappers.ColorMapper`], optional If a single label is used, then this can be a color mapper, if multiple labels are used, then this should be a Dict of label name -> color mapper. Used to set colors for the labels defined using ``label``. @@ -1103,7 +1165,7 @@ def joint_plot( Plot height collect_all : bool Whether to collect all values or downsample before plotting. - n_divisions : int + n_divisions : int, optional Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints. missing_label: str Label to use when a point is missing data for a categorical label @@ -1111,22 +1173,41 @@ def joint_plot( Returns ------- - :class:`bokeh.models.layouts.Column` + :class:`.GridPlot` """ # Collect data hover_fields = {} if hover_fields is None else hover_fields - label = {} if label is None else {'label': label} if isinstance(label, Expression) else label - colors = {'label': colors} if isinstance(colors, ColorMapper) else colors + + label_by_col: Dict[str, Expression] + if label is None: + label_by_col = {} + elif isinstance(label, Expression): + label_by_col = {'label': label} + else: + assert isinstance(label, dict) + label_by_col = label + + if isinstance(colors, ColorMapper): + colors_by_col = {'label': colors} + else: + colors_by_col = colors if isinstance(x, NumericExpression): - x = ('x', x) + _x = ('x', x) + else: + _x = x if isinstance(y, NumericExpression): - y = ('y', y) + _y = ('y', y) + else: + _y = y - label_cols = list(label.keys()) - source_pd = _collect_scatter_plot_data(x, y, fields={**hover_fields, **label}, n_divisions=None if collect_all else None, missing_label=missing_label) + label_cols = list(label_by_col.keys()) + source_pd = _collect_scatter_plot_data(_x, _y, fields={**hover_fields, **label_by_col}, n_divisions=None if collect_all else None, missing_label=missing_label) sp = figure(title=title, x_axis_label=xlabel, y_axis_label=ylabel, height=height, width=width) - sp, sp_legend_items, sp_legend, sp_color_bar, sp_color_mappers, sp_scatter_renderers = _get_scatter_plot_elements(sp, source_pd, x[0], y[0], label_cols, colors, size) + sp, sp_legend_items, sp_legend, sp_color_bar, sp_color_mappers, sp_scatter_renderers = _get_scatter_plot_elements( + sp, source_pd, _x[0], _y[0], label_cols, colors_by_col, size, + hover_cols={'x', 'y'} | set(hover_fields) + ) continuous_cols = [col for col in label_cols if (str(source_pd.dtypes[col]).startswith('float') @@ -1139,7 +1220,7 @@ def get_density_plot_items( data_col, p, x_axis, - colors: Dict[str, ColorMapper], + colors: Optional[Dict[str, ColorMapper]], continuous_cols: List[str], factor_cols: List[str] ): @@ -1156,6 +1237,7 @@ def get_density_plot_items( max_densities = {col: np.max(dens) for col in continuous_cols} for factor_col in factor_cols: + assert colors is not None, (colors, factor_cols) factor_colors = colors.get(factor_col, _get_categorical_palette(list(set(source_pd[factor_col])))) factor_colors = dict(zip(factor_colors.factors, factor_colors.palette)) density_data = source_pd[[factor_col, data_col]].groupby(factor_col).apply(lambda df: np.histogram(df['x' if x_axis else 'y'], density=True)) @@ -1172,22 +1254,24 @@ def get_density_plot_items( return p, density_renderers, max_densities xp = figure(title=title, height=int(height / 3), width=width, x_range=sp.x_range) - xp, x_renderers, x_max_densities = get_density_plot_items(source_pd, x[0], xp, x_axis=True, colors=sp_color_mappers, continuous_cols=continuous_cols, factor_cols=factor_cols) + xp, x_renderers, x_max_densities = get_density_plot_items(source_pd, _x[0], xp, x_axis=True, colors=sp_color_mappers, continuous_cols=continuous_cols, factor_cols=factor_cols) xp.xaxis.visible = False yp = figure(height=height, width=int(width / 3), y_range=sp.y_range) - yp, y_renderers, y_max_densities = get_density_plot_items(source_pd, y[0], yp, x_axis=False, colors=sp_color_mappers, continuous_cols=continuous_cols, factor_cols=factor_cols) + yp, y_renderers, y_max_densities = get_density_plot_items(source_pd, _y[0], yp, x_axis=False, colors=sp_color_mappers, continuous_cols=continuous_cols, factor_cols=factor_cols) yp.yaxis.visible = False density_renderers = x_renderers + y_renderers first_row = [xp] if not legend: + assert sp_legend is not None + assert sp_color_bar is not None sp_legend.visible = False sp_color_bar.visible = False # If multiple labels, create JS call back selector if len(label_cols) > 1: - for factor_col, factor, renderer in density_renderers: + for factor_col, _, renderer in density_renderers: renderer.visible = factor_col == label_cols[0] if label_cols[0] in factor_cols: @@ -1196,6 +1280,7 @@ def get_density_plot_items( yp.x_range.start = 0 yp.x_range.end = y_max_densities[label_cols[0]] + callback_args: Dict[str, Any] callback_args = dict( scatter_renderers=sp_scatter_renderers, color_mappers=sp_color_mappers, @@ -1250,38 +1335,34 @@ def get_density_plot_items( return gridplot([first_row, [sp, yp]]) -@typecheck(pvals=oneof(expr_numeric, sized_tupleof(str, expr_numeric)), +@typecheck(pvals=expr_numeric, label=nullable(oneof(dictof(str, expr_any), expr_any)), title=nullable(str), xlabel=nullable(str), ylabel=nullable(str), size=int, legend=bool, hover_fields=nullable(dictof(str, expr_any)), colors=nullable(oneof(bokeh.models.mappers.ColorMapper, dictof(str, bokeh.models.mappers.ColorMapper))), width=int, height=int, collect_all=bool, n_divisions=nullable(int), missing_label=str) def qq( - pvals: Union[NumericExpression, Tuple[str, NumericExpression]], - label: Union[Expression, Dict[str, Expression]] = None, - title: str = 'Q-Q plot', - xlabel: str = 'Expected -log10(p)', - ylabel: str = 'Observed -log10(p)', + pvals: NumericExpression, + label: Optional[Union[Expression, Dict[str, Expression]]] = None, + title: Optional[str] = 'Q-Q plot', + xlabel: Optional[str] = 'Expected -log10(p)', + ylabel: Optional[str] = 'Observed -log10(p)', size: int = 6, legend: bool = True, - hover_fields: Dict[str, Expression] = None, - colors: Union[ColorMapper, Dict[str, ColorMapper]] = None, + hover_fields: Optional[Dict[str, Expression]] = None, + colors: Optional[Union[ColorMapper, Dict[str, ColorMapper]]] = None, width: int = 800, height: int = 800, collect_all: bool = False, - n_divisions: int = 500, + n_divisions: Optional[int] = 500, missing_label: str = 'NA' -) -> Union[Figure, Column]: +) -> Union[figure, Column]: """Create a Quantile-Quantile plot. (https://en.wikipedia.org/wiki/Q-Q_plot) - ``pvals`` must be either: - - a :class:`.NumericExpression` - - a tuple (str, :class:`.NumericExpression`). If passed as a tuple the first element is used as the hover label. - - If no label or a single label is provided, then returns :class:`bokeh.plotting.figure.Figure` + If no label or a single label is provided, then returns :class:`bokeh.plotting.figure` Otherwise returns a :class:`bokeh.models.layouts.Column` containing: - a :class:`bokeh.models.widgets.inputs.Select` dropdown selection widget for labels - - a :class:`bokeh.plotting.figure.Figure` containing the interactive qq plot + - a :class:`bokeh.plotting.figure` containing the interactive qq plot Points will be colored by one of the labels defined in the ``label`` using the color scheme defined in the corresponding entry of ``colors`` if provided (otherwise a default scheme is used). To specify your color @@ -1296,7 +1377,7 @@ def qq( Parameters ---------- - pvals : :class:`.NumericExpression` or (str, :class:`.NumericExpression`) + pvals : :class:`.NumericExpression` List of x-values to be plotted. label : :class:`.Expression` or Dict[str, :class:`.Expression`]] Either a single expression (if a single label is desired), or a @@ -1304,19 +1385,19 @@ def qq( Used to color each point w.r.t its label. When multiple labels are given, a dropdown will be displayed with the different options. Can be used with categorical or continuous expressions. - title : str + title : str, optional Title of the scatterplot. - xlabel : str + xlabel : str, optional X-axis label. - ylabel : str + ylabel : str, optional Y-axis label. size : int Size of markers in screen space units. legend: bool Whether or not to show the legend in the resulting figure. - hover_fields : Dict[str, :class:`.Expression`] + hover_fields : Dict[str, :class:`.Expression`], optional Extra fields to be displayed when hovering over a point on the plot. - colors : :class:`bokeh.models.mappers.ColorMapper` or Dict[str, :class:`bokeh.models.mappers.ColorMapper`] + colors : :class:`bokeh.models.mappers.ColorMapper` or Dict[str, :class:`bokeh.models.mappers.ColorMapper`], optional If a single label is used, then this can be a color mapper, if multiple labels are used, then this should be a Dict of label name -> color mapper. Used to set colors for the labels defined using ``label``. @@ -1327,23 +1408,32 @@ def qq( Plot height collect_all : bool Whether to collect all values or downsample before plotting. - n_divisions : int + n_divisions : int, optional Factor by which to downsample (default value = 500). A lower input results in fewer output datapoints. missing_label: str Label to use when a point is missing data for a categorical label Returns ------- - :class:`bokeh.plotting.figure.Figure` if no label or a single label was given, otherwise :class:`bokeh.models.layouts.Column` + :class:`bokeh.plotting.figure` if no label or a single label was given, otherwise :class:`bokeh.models.layouts.Column` """ hover_fields = {} if hover_fields is None else hover_fields - label = {} if label is None else {'label': label} if isinstance(label, Expression) else label + label_by_col: Dict[str, Expression] + if label is None: + label_by_col = {} + elif isinstance(label, Expression): + label_by_col = {'label': label} + else: + assert isinstance(label, dict) + label_by_col = label + source = pvals._indices.source if isinstance(source, Table): - ht = source.select(p_value=pvals, **hover_fields, **label) + ht = source.select(p_value=pvals, **hover_fields, **label_by_col) else: - ht = source.select_rows(p_value=pvals, **hover_fields, **label).rows() - ht = ht.key_by().select('p_value', *hover_fields, *label).key_by('p_value') + assert isinstance(source, MatrixTable) + ht = source.select_rows(p_value=pvals, **hover_fields, **label_by_col).rows() + ht = ht.key_by().select('p_value', *hover_fields, *label_by_col).key_by('p_value') n = ht.aggregate(aggregators.count(), _localize=False) ht = ht.annotate( observed_p=-hail.log10(ht['p_value']), @@ -1354,7 +1444,7 @@ def qq( p = scatter( ht.expected_p, ht.observed_p, - label={x: ht[x] for x in label}, + label={x: ht[x] for x in label_by_col}, title=title, xlabel=xlabel, ylabel=ylabel, @@ -1389,7 +1479,7 @@ def qq( @typecheck(pvals=expr_float64, locus=nullable(expr_locus()), title=nullable(str), size=int, hover_fields=nullable(dictof(str, expr_any)), collect_all=bool, n_divisions=int, significance_line=nullable(numeric)) -def manhattan(pvals, locus=None, title=None, size=4, hover_fields=None, collect_all=False, n_divisions=500, significance_line=5e-8) -> Figure: +def manhattan(pvals, locus=None, title=None, size=4, hover_fields=None, collect_all=False, n_divisions=500, significance_line=5e-8) -> Plot: """Create a Manhattan plot. (https://en.wikipedia.org/wiki/Manhattan_plot) Parameters @@ -1414,7 +1504,7 @@ def manhattan(pvals, locus=None, title=None, size=4, hover_fields=None, collect_ Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.models.Plot` """ if locus is None: locus = pvals._indices.source.locus @@ -1437,22 +1527,29 @@ def manhattan(pvals, locus=None, title=None, size=4, hover_fields=None, collect_ source_pd['p_value'] = [10 ** (-p) for p in source_pd['_pval']] source_pd['_contig'] = [locus.split(":")[0] for locus in source_pd['locus']] - observed_contigs = set(source_pd['_contig']) - observed_contigs = [contig for contig in ref.contigs.copy() if contig in observed_contigs] + observed_contigs = [ + contig for contig in ref.contigs.copy() + if contig in set(source_pd['_contig']) + ] contig_ticks = [ref._contig_global_position(contig) + ref.contig_length(contig) // 2 for contig in observed_contigs] color_mapper = CategoricalColorMapper(factors=ref.contigs, palette=palette[:2] * int((len(ref.contigs) + 1) / 2)) p = figure(title=title, x_axis_label='Chromosome', y_axis_label='P-value (-log10 scale)', width=1000) p, _, legend, _, _, _ = _get_scatter_plot_elements( - p, source_pd, x_col='_global_locus', y_col='_pval', - label_cols=['_contig'], colors={'_contig': color_mapper}, - size=size + p, + source_pd, + x_col='_global_locus', + y_col='_pval', + label_cols=['_contig'], + colors={'_contig': color_mapper}, + size=size, + hover_cols={'locus', 'p_value'} | set(hover_fields) ) + assert legend is not None legend.visible = False p.xaxis.ticker = contig_ticks p.xaxis.major_label_overrides = dict(zip(contig_ticks, observed_contigs)) - p.select_one(HoverTool).tooltips = [t for t in p.select_one(HoverTool).tooltips if not t[0].startswith('_')] if significance_line is not None: p.renderers.append(Span(location=-math.log10(significance_line), @@ -1467,7 +1564,7 @@ def manhattan(pvals, locus=None, title=None, size=4, hover_fields=None, collect_ @typecheck(entry_field=expr_any, row_field=nullable(oneof(expr_numeric, expr_locus())), column_field=nullable(expr_str), window=nullable(int), plot_width=int, plot_height=int) def visualize_missingness(entry_field, row_field=None, column_field=None, - window=6000000, plot_width=1800, plot_height=900) -> Figure: + window=6000000, plot_width=1800, plot_height=900) -> figure: """Visualize missingness in a MatrixTable. Inspired by `naniar `__. @@ -1497,7 +1594,7 @@ def visualize_missingness(entry_field, row_field=None, column_field=None, Returns ------- - :class:`bokeh.plotting.figure.Figure` + :class:`bokeh.plotting.figure` """ mt = entry_field._indices.source if row_field is None: @@ -1551,9 +1648,8 @@ def visualize_missingness(entry_field, row_field=None, column_field=None, df = pd.DataFrame(df.stack(), columns=['defined']).reset_index() - from bokeh.plotting import figure p = figure(x_range=columns, y_range=list(reversed(rows)), - x_axis_location="above", plot_width=plot_width, plot_height=plot_height, + x_axis_location="above", width=plot_width, height=plot_height, toolbar_location='below', tooltips=[('defined', '@defined'), ('row', '@row'), ('column', '@column')] ) diff --git a/hail/python/hailtop/batch/backend.py b/hail/python/hailtop/batch/backend.py index be4b19a588d..4c96ce547d3 100644 --- a/hail/python/hailtop/batch/backend.py +++ b/hail/python/hailtop/batch/backend.py @@ -630,9 +630,9 @@ def symlink_input_resource_group(r): for pyjob in pyjobs: if pyjob._image is None: version = sys.version_info - if version.major != 3 or version.minor not in (7, 8, 9, 10): + if version.major != 3 or version.minor not in (8, 9, 10): raise BatchException( - f"You must specify 'image' for Python jobs if you are using a Python version other than 3.7, 3.8, 3.9 or 3.10 (you are using {version})") + f"You must specify 'image' for Python jobs if you are using a Python version other than 3.8, 3.9 or 3.10 (you are using {version})") pyjob._image = f'hailgenetics/python-dill:{version.major}.{version.minor}-slim' await batch._serialize_python_functions_to_input_files( diff --git a/hail/python/hailtop/batch/batch.py b/hail/python/hailtop/batch/batch.py index a70a24234ac..05dd5137acf 100644 --- a/hail/python/hailtop/batch/batch.py +++ b/hail/python/hailtop/batch/batch.py @@ -16,8 +16,7 @@ class Batch: - """ - Object representing the distributed acyclic graph (DAG) of jobs to run. + """Object representing the distributed acyclic graph (DAG) of jobs to run. Examples -------- @@ -81,12 +80,11 @@ class Batch: applicable for the :class:`.ServiceBackend`. If `None`, there is no timeout. default_python_image: - Default image to use for all Python jobs. This must be the full name of the - image including any repository prefix and tags if desired (default tag is `latest`). - The image must have the `dill` Python package installed and have the same version of - Python installed that is currently running. If `None`, a compatible Python image with - `dill` pre-installed will automatically be used if the current Python version is - 3.6, 3.7, or 3.8. + Default image to use for all Python jobs. This must be the full name of the image including + any repository prefix and tags if desired (default tag is `latest`). The image must have + the `dill` Python package installed and have the same version of Python installed that is + currently running. If `None`, a compatible Python image with `dill` pre-installed will + automatically be used if the current Python version is 3.8, 3.9, or 3.10. project: DEPRECATED: please specify `google_project` on the ServiceBackend instead. If specified, the project to use when authenticating with Google Storage. Google Storage is used to @@ -96,6 +94,7 @@ class Batch: Automatically cancel the batch after N failures have occurred. The default behavior is there is no limit on the number of failures. Only applicable for the :class:`.ServiceBackend`. Must be greater than 0. + """ _counter = 0 @@ -331,7 +330,7 @@ def new_python_job(self, .. code-block:: python - b = Batch(default_python_image='hailgenetics/python-dill:3.7-slim') + b = Batch(default_python_image='hailgenetics/python-dill:3.8-slim') def hello(name): return f'hello {name}' diff --git a/hail/python/hailtop/batch/batch_pool_executor.py b/hail/python/hailtop/batch/batch_pool_executor.py index eb9ea1e1bfb..027272d335c 100644 --- a/hail/python/hailtop/batch/batch_pool_executor.py +++ b/hail/python/hailtop/batch/batch_pool_executor.py @@ -93,7 +93,7 @@ class BatchPoolExecutor: Backend used to execute the jobs. Must be a :class:`.ServiceBackend`. image: The name of a Docker image used for each submitted job. The image must - include Python 3.7 or later and must have the ``dill`` Python package + include Python 3.8 or later and must have the ``dill`` Python package installed. If you intend to use ``numpy``, ensure that OpenBLAS is also installed. If unspecified, an image with a matching Python verison and ``numpy``, ``scipy``, and ``sklearn`` installed is used. @@ -138,9 +138,9 @@ def __init__(self, *, self._shutdown = False version = sys.version_info if image is None: - if version.major != 3 or version.minor not in (7, 8, 9, 10): + if version.major != 3 or version.minor not in (8, 9, 10): raise ValueError( - f'You must specify an image if you are using a Python version other than 3.7, 3.8, 3.9 or 3.10 (you are using {version})') + f'You must specify an image if you are using a Python version other than 3.8, 3.9 or 3.10 (you are using {version})') self.image = f'hailgenetics/python-dill:{version.major}.{version.minor}-slim' else: self.image = image diff --git a/hail/python/hailtop/batch/docker.py b/hail/python/hailtop/batch/docker.py index 9ddb1ac1dfe..d5111609cf9 100644 --- a/hail/python/hailtop/batch/docker.py +++ b/hail/python/hailtop/batch/docker.py @@ -34,7 +34,7 @@ def build_python_image(fullname: str, requirements: List of pip packages to install. python_version: - String in the format of `major_version.minor_version` (ex: `3.7`). Defaults to + String in the format of `major_version.minor_version` (ex: `3.8`). Defaults to current version of Python that is running. _tmp_dir: Location to place local temporary files used while building the image. @@ -54,9 +54,9 @@ def build_python_image(fullname: str, major_version = int(version[0]) minor_version = int(version[1]) - if major_version != 3 or minor_version < 7: + if major_version != 3 or minor_version < 8: raise ValueError( - f'Python versions older than 3.7 (you are using {major_version}.{minor_version}) are not supported') + f'Python versions older than 3.8 (you are using {major_version}.{minor_version}) are not supported') base_image = f'hailgenetics/python-dill:{major_version}.{minor_version}-slim' diff --git a/hail/python/hailtop/batch/docs/change_log.rst b/hail/python/hailtop/batch/docs/change_log.rst index 67bf0e8ffa7..aa5ceabbd59 100644 --- a/hail/python/hailtop/batch/docs/change_log.rst +++ b/hail/python/hailtop/batch/docs/change_log.rst @@ -1,5 +1,17 @@ .. _sec-change-log: +Python Version Compatibility Policy +=================================== + +Hail complies with [NumPy's compatibility policy](https://numpy.org/neps/nep-0029-deprecation_policy.html#implementation) on Python +versions. In particular, Hail officially supports: + +- All minor versions of Python released 42 months prior to the project, and at minimum the two + latest minor versions. + +- All minor versions of numpy released in the 24 months prior to the project, and at minimum the + last three minor versions. + Change Log ========== diff --git a/hail/python/hailtop/batch/docs/conf.py b/hail/python/hailtop/batch/docs/conf.py index a876394f22a..258ce5834b5 100644 --- a/hail/python/hailtop/batch/docs/conf.py +++ b/hail/python/hailtop/batch/docs/conf.py @@ -47,6 +47,7 @@ 'sphinx.ext.napoleon', 'sphinx_autodoc_typehints', 'IPython.sphinxext.ipython_console_highlighting', + 'sphinx_rtd_theme', ] automodapi_inheritance_diagram = False @@ -77,7 +78,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -117,7 +118,7 @@ # html_sidebars = {} # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html -intersphinx_mapping = {'python': ('https://docs.python.org/3.7', None)} +intersphinx_mapping = {'python': ('https://docs.python.org/3.8', None)} # -- Extension configuration ------------------------------------------------- diff --git a/hail/python/hailtop/batch/docs/cookbook/random_forest.rst b/hail/python/hailtop/batch/docs/cookbook/random_forest.rst index 144297bcb35..0fdb2a391e1 100644 --- a/hail/python/hailtop/batch/docs/cookbook/random_forest.rst +++ b/hail/python/hailtop/batch/docs/cookbook/random_forest.rst @@ -120,7 +120,7 @@ Build Python Image In order to run a :class:`.PythonJob`, Batch needs an image that has the same version of Python as the version of Python running on your computer and the Python package `dill` installed. Batch will automatically -choose a suitable image for you if your Python version is 3.7 or newer. +choose a suitable image for you if your Python version is 3.8 or newer. You can supply your own image that meets the requirements listed above to the method :meth:`.PythonJob.image` or as the argument `default_python_image` when constructing a Batch . We also provide a convenience function :func:`.docker.build_python_image` diff --git a/hail/python/hailtop/batch/docs/getting_started.rst b/hail/python/hailtop/batch/docs/getting_started.rst index 7c8df64e1e5..4f76a5b64e1 100644 --- a/hail/python/hailtop/batch/docs/getting_started.rst +++ b/hail/python/hailtop/batch/docs/getting_started.rst @@ -20,7 +20,7 @@ Create a `conda enviroment .. code-block:: sh - conda create -n hail python'>=3.7' + conda create -n hail python'>=3.8' conda activate hail pip install hail diff --git a/hail/python/hailtop/batch/docs/index.rst b/hail/python/hailtop/batch/docs/index.rst index 3b84f61e2aa..6650d3a38ac 100644 --- a/hail/python/hailtop/batch/docs/index.rst +++ b/hail/python/hailtop/batch/docs/index.rst @@ -22,7 +22,7 @@ Contents Batch Service Cookbooks Reference (Python API) - Change Log + Change Log And Version Policy Indices and tables diff --git a/hail/python/hailtop/batch/job.py b/hail/python/hailtop/batch/job.py index 5485a256d6f..ca43085e060 100644 --- a/hail/python/hailtop/batch/job.py +++ b/hail/python/hailtop/batch/job.py @@ -864,7 +864,7 @@ class PythonJob(Job): # Create a batch object with a default Python image - b = Batch(default_python_image='hailgenetics/python-dill:3.7-slim') + b = Batch(default_python_image='hailgenetics/python-dill:3.8-slim') def multiply(x, y): return x * y @@ -922,11 +922,11 @@ def image(self, image: str) -> 'PythonJob': Examples -------- - Set the job's docker image to `hailgenetics/python-dill:3.7-slim`: + Set the job's docker image to `hailgenetics/python-dill:3.8-slim`: >>> b = Batch() >>> j = b.new_python_job() - >>> (j.image('hailgenetics/python-dill:3.7-slim') + >>> (j.image('hailgenetics/python-dill:3.8-slim') ... .call(print, 'hello')) >>> b.run() # doctest: +SKIP diff --git a/hail/python/hailtop/pinned-requirements.txt b/hail/python/hailtop/pinned-requirements.txt index 1f0695d8ebd..7ffc0ae7fbe 100644 --- a/hail/python/hailtop/pinned-requirements.txt +++ b/hail/python/hailtop/pinned-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/hail/python/hailtop/pinned-requirements.txt hail/hail/python/hailtop/requirements.txt # @@ -12,9 +12,7 @@ aiosignal==1.3.1 # via aiohttp async-timeout==4.0.2 # via aiohttp -asynctest==0.13.0 - # via aiohttp -attrs==22.2.0 +attrs==23.1.0 # via aiohttp azure-core==1.26.4 # via @@ -22,11 +20,11 @@ azure-core==1.26.4 # azure-storage-blob azure-identity==1.12.0 # via -r hail/hail/python/hailtop/requirements.txt -azure-storage-blob==12.15.0 +azure-storage-blob==12.16.0 # via -r hail/hail/python/hailtop/requirements.txt -boto3==1.26.110 +boto3==1.26.118 # via -r hail/hail/python/hailtop/requirements.txt -botocore==1.29.110 +botocore==1.29.118 # via # -r hail/hail/python/hailtop/requirements.txt # boto3 @@ -45,7 +43,7 @@ charset-normalizer==3.1.0 # requests commonmark==0.9.1 # via rich -cryptography==40.0.1 +cryptography==40.0.2 # via # azure-identity # azure-storage-blob @@ -62,7 +60,7 @@ google-api-core==2.11.0 # via # google-cloud-core # google-cloud-storage -google-auth==2.17.2 +google-auth==2.17.3 # via # -r hail/hail/python/hailtop/requirements.txt # google-api-core @@ -74,7 +72,7 @@ google-cloud-storage==2.8.0 # via -r hail/hail/python/hailtop/requirements.txt google-crc32c==1.5.0 # via google-resumable-media -google-resumable-media==2.4.1 +google-resumable-media==2.5.0 # via google-cloud-storage googleapis-common-protos==1.59.0 # via google-api-core @@ -92,7 +90,7 @@ jmespath==1.0.1 # via # boto3 # botocore -msal==1.21.0 +msal==1.22.0 # via # azure-identity # msal-extensions @@ -113,11 +111,11 @@ protobuf==3.20.2 # -r hail/hail/python/hailtop/requirements.txt # google-api-core # googleapis-common-protos -pyasn1==0.4.8 +pyasn1==0.5.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.2.8 +pyasn1-modules==0.3.0 # via google-auth pycares==4.3.0 # via aiodns @@ -156,18 +154,15 @@ tabulate==0.9.0 # via -r hail/hail/python/hailtop/requirements.txt typing-extensions==4.5.0 # via - # aiohttp - # async-timeout # azure-core # azure-storage-blob # janus # rich - # yarl urllib3==1.26.15 # via # botocore # requests uvloop==0.17.0 ; sys_platform != "win32" # via -r hail/hail/python/hailtop/requirements.txt -yarl==1.8.2 +yarl==1.9.1 # via aiohttp diff --git a/hail/python/pinned-requirements.txt b/hail/python/pinned-requirements.txt index b4d62604c17..44eebf2328b 100644 --- a/hail/python/pinned-requirements.txt +++ b/hail/python/pinned-requirements.txt @@ -1,58 +1,85 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/hail/python/pinned-requirements.txt hail/hail/python/requirements.txt # aiodns==2.0.0 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt aiohttp==3.8.4 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt aiosignal==1.3.1 - # via aiohttp + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # aiohttp async-timeout==4.0.2 - # via aiohttp + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # aiohttp asyncinit==0.2.4 # via -r hail/hail/python/requirements.txt -asynctest==0.13.0 - # via aiohttp -attrs==22.2.0 - # via aiohttp +attrs==23.1.0 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # aiohttp avro==1.11.1 # via -r hail/hail/python/requirements.txt azure-core==1.26.4 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # azure-identity # azure-storage-blob azure-identity==1.12.0 - # via -r hail/hail/python/hailtop/requirements.txt -azure-storage-blob==12.15.0 - # via -r hail/hail/python/hailtop/requirements.txt -bokeh==1.4.0 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt +azure-storage-blob==12.16.0 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt +bokeh==3.1.0 # via -r hail/hail/python/requirements.txt -boto3==1.26.110 - # via -r hail/hail/python/hailtop/requirements.txt -botocore==1.29.110 +boto3==1.26.118 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt +botocore==1.29.118 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # -r hail/hail/python/hailtop/requirements.txt # boto3 # s3transfer cachetools==5.3.0 - # via google-auth + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # google-auth certifi==2022.12.7 - # via requests + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # requests cffi==1.15.1 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # cryptography # pycares charset-normalizer==3.1.0 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # aiohttp # requests commonmark==0.9.1 - # via rich -cryptography==40.0.1 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # rich +contourpy==1.0.7 + # via bokeh +cryptography==40.0.2 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # azure-identity # azure-storage-blob # msal @@ -63,82 +90,113 @@ deprecated==1.2.13 # via -r hail/hail/python/requirements.txt dill==0.3.6 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # -r hail/hail/python/hailtop/requirements.txt # -r hail/hail/python/requirements.txt frozenlist==1.3.3 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # -r hail/hail/python/hailtop/requirements.txt # -r hail/hail/python/requirements.txt # aiohttp # aiosignal google-api-core==2.11.0 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # google-cloud-core # google-cloud-storage -google-auth==2.17.2 +google-auth==2.17.3 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # -r hail/hail/python/hailtop/requirements.txt # google-api-core # google-cloud-core # google-cloud-storage google-cloud-core==2.3.2 - # via google-cloud-storage + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # google-cloud-storage google-cloud-storage==2.8.0 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt google-crc32c==1.5.0 - # via google-resumable-media -google-resumable-media==2.4.1 - # via google-cloud-storage + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # google-resumable-media +google-resumable-media==2.5.0 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # google-cloud-storage googleapis-common-protos==1.59.0 - # via google-api-core + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # google-api-core humanize==1.1.0 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt hurry-filesize==0.9 # via -r hail/hail/python/requirements.txt idna==3.4 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # requests # yarl isodate==0.6.1 - # via azure-storage-blob + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # azure-storage-blob janus==1.0.0 - # via -r hail/hail/python/hailtop/requirements.txt -jinja2==3.0.3 # via - # -r hail/hail/python/requirements.txt - # bokeh + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt +jinja2==3.1.2 + # via bokeh jmespath==1.0.1 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # boto3 # botocore markupsafe==2.1.2 # via jinja2 -msal==1.21.0 +msal==1.22.0 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # azure-identity # msal-extensions msal-extensions==1.0.0 - # via azure-identity + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # azure-identity multidict==6.0.4 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # aiohttp # yarl nest-asyncio==1.5.6 - # via -r hail/hail/python/hailtop/requirements.txt -numpy==1.21.6 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt +numpy==1.24.3 # via # -r hail/hail/python/requirements.txt # bokeh + # contourpy # pandas # scipy orjson==3.8.10 - # via -r hail/hail/python/hailtop/requirements.txt -packaging==23.0 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt +packaging==23.1 # via # bokeh # plotly -pandas==1.3.5 - # via -r hail/hail/python/requirements.txt +pandas==2.0.1 + # via + # -r hail/hail/python/requirements.txt + # bokeh parsimonious==0.10.0 # via -r hail/hail/python/requirements.txt pillow==9.5.0 @@ -146,95 +204,128 @@ pillow==9.5.0 plotly==5.14.1 # via -r hail/hail/python/requirements.txt portalocker==2.7.0 - # via msal-extensions + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # msal-extensions protobuf==3.20.2 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # -r hail/hail/python/hailtop/requirements.txt # -r hail/hail/python/requirements.txt # google-api-core # googleapis-common-protos py4j==0.10.9.5 # via pyspark -pyasn1==0.4.8 +pyasn1==0.5.0 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # pyasn1-modules # rsa -pyasn1-modules==0.2.8 - # via google-auth +pyasn1-modules==0.3.0 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # google-auth pycares==4.3.0 - # via aiodns + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # aiodns pycparser==2.21 - # via cffi + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # cffi pygments==2.15.1 - # via rich + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # rich pyjwt[crypto]==2.6.0 - # via msal + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # msal pyspark==3.3.2 # via -r hail/hail/python/requirements.txt python-dateutil==2.8.2 # via - # bokeh + # -c hail/hail/python/hailtop/pinned-requirements.txt # botocore # pandas python-json-logger==2.0.7 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt pytz==2023.3 # via pandas pyyaml==6.0 # via bokeh -regex==2022.10.31 +regex==2023.3.23 # via parsimonious requests==2.28.2 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # -r hail/hail/python/requirements.txt # azure-core # google-api-core # google-cloud-storage # msal rich==12.6.0 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt rsa==4.9 - # via google-auth + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # google-auth s3transfer==0.6.0 - # via boto3 -scipy==1.7.3 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # boto3 +scipy==1.9.3 # via -r hail/hail/python/requirements.txt six==1.16.0 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # azure-core # azure-identity - # bokeh # google-auth # isodate # python-dateutil sortedcontainers==2.4.0 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt tabulate==0.9.0 - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt tenacity==8.2.2 # via plotly -tornado==6.2 +tornado==6.3.1 # via bokeh typing-extensions==4.5.0 # via - # aiohttp - # async-timeout - # avro + # -c hail/hail/python/hailtop/pinned-requirements.txt # azure-core # azure-storage-blob # janus # rich - # yarl +tzdata==2023.3 + # via pandas urllib3==1.26.15 # via + # -c hail/hail/python/hailtop/pinned-requirements.txt # botocore # requests uvloop==0.17.0 ; sys_platform != "win32" - # via -r hail/hail/python/hailtop/requirements.txt + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # -r hail/hail/python/hailtop/requirements.txt wrapt==1.15.0 # via deprecated -yarl==1.8.2 - # via aiohttp +xyzservices==2023.2.0 + # via bokeh +yarl==1.9.1 + # via + # -c hail/hail/python/hailtop/pinned-requirements.txt + # aiohttp # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/hail/python/requirements.txt b/hail/python/requirements.txt index eebf5f8232f..56065fc4855 100644 --- a/hail/python/requirements.txt +++ b/hail/python/requirements.txt @@ -1,15 +1,16 @@ +-c hailtop/pinned-requirements.txt -r hailtop/requirements.txt + asyncinit>=0.2.4,<0.3 avro>=1.10,<1.12 -bokeh==1.4.0 +bokeh>=3,<4 decorator<5 Deprecated>=1.2.10,<1.3 dill>=0.3.1.1,<0.4 frozenlist>=1.3.1,<2 hurry.filesize>=0.9,<1 -Jinja2==3.0.3 numpy<2 -pandas>=1.3.0,<2.1 +pandas>=2,<3 parsimonious<1 plotly>=5.5.0,<6 protobuf==3.20.2 diff --git a/hail/python/setup.py b/hail/python/setup.py index 574ce615bd1..66bbfae6e12 100755 --- a/hail/python/setup.py +++ b/hail/python/setup.py @@ -25,6 +25,8 @@ def add_dependencies(fname): stripped = line.strip() if stripped.startswith('#') or len(stripped) == 0: continue + if stripped.startswith('-c'): + continue if stripped.startswith('-r'): additional_requirements = stripped[len('-r'):].strip() add_dependencies(additional_requirements) @@ -68,7 +70,7 @@ def add_dependencies(fname): "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", ], - python_requires=">=3.7", + python_requires=">=3.8", install_requires=dependencies, entry_points={ 'console_scripts': ['hailctl = hailtop.hailctl.__main__:main'] diff --git a/hail/python/test/hail/expr/test_expr.py b/hail/python/test/hail/expr/test_expr.py index 1ddbe72f1a9..87830b5eb44 100644 --- a/hail/python/test/hail/expr/test_expr.py +++ b/hail/python/test/hail/expr/test_expr.py @@ -3745,11 +3745,11 @@ def test_tuple_slice(self): def test_numpy_conversions(self): assert hl.eval(np.int32(3)) == 3 assert hl.eval(np.int64(1234)) == 1234 - assert hl.eval(np.bool(True)) - assert not hl.eval(np.bool(False)) + assert hl.eval(np.bool_(True)) + assert not hl.eval(np.bool_(False)) assert np.allclose(hl.eval(np.float32(3.4)), 3.4) assert np.allclose(hl.eval(np.float64(8.89)), 8.89) - assert hl.eval(np.str("cat")) == "cat" + assert hl.eval(np.str_("cat")) == "cat" def test_array_struct_error(self): a = hl.array([hl.struct(a=5)]) diff --git a/hail/python/test/hailtop/batch/test_batch.py b/hail/python/test/hailtop/batch/test_batch.py index 790c8d29af6..d94bc910956 100644 --- a/hail/python/test/hailtop/batch/test_batch.py +++ b/hail/python/test/hailtop/batch/test_batch.py @@ -22,7 +22,7 @@ DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:20.04') -PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.7-slim' +PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.8-slim' HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') diff --git a/hail/python/test/hailtop/batch/test_batch_pool_executor.py b/hail/python/test/hailtop/batch/test_batch_pool_executor.py index 6b3437a27a2..901b4cc2cc4 100644 --- a/hail/python/test/hailtop/batch/test_batch_pool_executor.py +++ b/hail/python/test/hailtop/batch/test_batch_pool_executor.py @@ -1,5 +1,4 @@ import asyncio -import concurrent.futures import time import pytest @@ -9,7 +8,7 @@ from hailtop.utils import sync_sleep_and_backoff from hailtop.batch_client.client import BatchClient -PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.7' +PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.8' submitted_batch_ids = [] @@ -141,7 +140,7 @@ def sleep_forever(): time.sleep(3600) try: list(bpe.map(lambda _: sleep_forever(), range(5), timeout=2)) - except concurrent.futures.TimeoutError: + except asyncio.TimeoutError: pass else: assert False diff --git a/memory/pinned-requirements.txt b/memory/pinned-requirements.txt index c1b0dc4b356..6d258ce848d 100644 --- a/memory/pinned-requirements.txt +++ b/memory/pinned-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/memory/pinned-requirements.txt hail/memory/requirements.txt # @@ -13,9 +13,3 @@ async-timeout==4.0.2 # aioredis hiredis==2.2.2 # via aioredis -typing-extensions==4.5.0 - # via - # -c hail/memory/../gear/pinned-requirements.txt - # -c hail/memory/../hail/python/dev/pinned-requirements.txt - # -c hail/memory/../hail/python/pinned-requirements.txt - # async-timeout diff --git a/web_common/pinned-requirements.txt b/web_common/pinned-requirements.txt index 280a6a90591..870f311854e 100644 --- a/web_common/pinned-requirements.txt +++ b/web_common/pinned-requirements.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # pip-compile --output-file=hail/web_common/pinned-requirements.txt hail/web_common/requirements.txt # @@ -21,12 +21,7 @@ async-timeout==4.0.2 # -c hail/web_common/../gear/pinned-requirements.txt # -c hail/web_common/../hail/python/pinned-requirements.txt # aiohttp -asynctest==0.13.0 - # via - # -c hail/web_common/../gear/pinned-requirements.txt - # -c hail/web_common/../hail/python/pinned-requirements.txt - # aiohttp -attrs==22.2.0 +attrs==23.1.0 # via # -c hail/web_common/../gear/pinned-requirements.txt # -c hail/web_common/../hail/python/dev/pinned-requirements.txt @@ -50,7 +45,7 @@ idna==3.4 # -c hail/web_common/../hail/python/dev/pinned-requirements.txt # -c hail/web_common/../hail/python/pinned-requirements.txt # yarl -jinja2==3.0.3 +jinja2==3.1.2 # via # -c hail/web_common/../hail/python/dev/pinned-requirements.txt # -c hail/web_common/../hail/python/pinned-requirements.txt @@ -69,16 +64,7 @@ multidict==6.0.4 # -c hail/web_common/../hail/python/pinned-requirements.txt # aiohttp # yarl -typing-extensions==4.5.0 - # via - # -c hail/web_common/../gear/pinned-requirements.txt - # -c hail/web_common/../hail/python/dev/pinned-requirements.txt - # -c hail/web_common/../hail/python/pinned-requirements.txt - # aiohttp - # aiohttp-jinja2 - # async-timeout - # yarl -yarl==1.8.2 +yarl==1.9.1 # via # -c hail/web_common/../gear/pinned-requirements.txt # -c hail/web_common/../hail/python/pinned-requirements.txt diff --git a/web_common/requirements.txt b/web_common/requirements.txt index ddecbc15733..b0394ce0db1 100644 --- a/web_common/requirements.txt +++ b/web_common/requirements.txt @@ -2,5 +2,5 @@ -c ../hail/python/dev/pinned-requirements.txt -c ../gear/pinned-requirements.txt aiohttp-jinja2>=1.1.1,<2 -Jinja2==3.0.3 +Jinja2>3,<4 libsass>=0.19.2,<1 diff --git a/website/Makefile b/website/Makefile index 44726198e9b..a62f29b543b 100644 --- a/website/Makefile +++ b/website/Makefile @@ -9,7 +9,7 @@ build: run: $(MAKE) -C .. docs - cd website && tar -xvzf ../docs.tar.gz + cd website && tar -xvzf ../../docs.tar.gz HAIL_DOMAIN=localhost:5000 python3 -m website local run-docker: build From 54a7f602e959e43e508d8e8c9d2d18dd96d39f6b Mon Sep 17 00:00:00 2001 From: Dan King Date: Tue, 2 May 2023 22:55:06 -0400 Subject: [PATCH 15/26] [qob] use a regional bucket with uniform access control (#12969) The old bucket did not use uniform access control and also was multi-regional (us). I created a new bucket using the random suffix ger0g which has uniform access control. I also switched the location to us-central1 (not pictured here because that is a variable). I copied all the JARs from `gs://hail-query/jars` to `gs://hail-query-ger0g/jars` using a GCE VM. Again, global-config is not present in our terraform, so I'll have to manually edit that to reflect this new location: `gs://hail-query-ger0g`. The deployment process is: 1. Edit global-config to reflect new bucket. 2. Delete batch and batch-driver pods. 3. Delete old workers. The rollback process (if necessary) is the same. Since this requires wiping the workers, I'll wait for a time when no one is on the cluster to do it. Any users using explicit JAR URLs will need to switch to `gs://hail-query-ger0g/...`. --- infra/gcp-broad/main.tf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/infra/gcp-broad/main.tf b/infra/gcp-broad/main.tf index 94faa34b4c6..b208cfe5896 100644 --- a/infra/gcp-broad/main.tf +++ b/infra/gcp-broad/main.tf @@ -585,12 +585,17 @@ resource "google_storage_bucket" "batch_logs" { } +resource "random_string" "hail_query_bucket_suffix" { + length = 5 +} + resource "google_storage_bucket" "hail_query" { - name = "hail-query" + name = "hail-query-${random_string.hail_query_bucket_suffix.result}" location = var.hail_query_bucket_location storage_class = var.hail_query_bucket_storage_class + uniform_bucket_level_access = true labels = { - "name" = "hail-query" + "name" = "hail-query-${random_string.hail_query_bucket_suffix.result}" } timeouts {} } From 1e4bdaddfe78dcf995efbde25ba9aae89394c96b Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Wed, 3 May 2023 02:01:54 -0400 Subject: [PATCH 16/26] [batch] Restrict mount propagation for job container mounts (#12960) Applies the most restrictive bind and event propagation settings to job container mounts. While user jobs do not have the capabilities to create mount points, overlapping mount points in the container config can inadvertently trigger mount propagation back to the host which we just never want. --- batch/batch/worker/worker.py | 37 +++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/batch/batch/worker/worker.py b/batch/batch/worker/worker.py index 935e36a422b..80d03d03d1f 100644 --- a/batch/batch/worker/worker.py +++ b/batch/batch/worker/worker.py @@ -1035,10 +1035,21 @@ async def _run_container(self) -> bool: return False + def _validate_container_config(self, config): + for mount in config['mounts']: + # bind mounts are given the dummy type 'none' + if mount['type'] == 'none': + # Mount events should not be propagated from the job container to the host + assert 'shared' not in mount['options'] + assert any(option in mount['options'] for option in ('private', 'slave')) + async def _write_container_config(self): + config = await self.container_config() + self._validate_container_config(config) + os.makedirs(self.config_path) with open(f'{self.config_path}/config.json', 'w', encoding='utf-8') as f: - f.write(json.dumps(await self.container_config())) + f.write(json.dumps(config)) # https://github.com/opencontainers/runtime-spec/blob/master/config.md async def container_config(self): @@ -1179,7 +1190,7 @@ def _mounts(self, uid, gid): 'source': v_host_path, 'destination': v_container_path, 'type': 'none', - 'options': ['rbind', 'rw', 'shared'], + 'options': ['bind', 'rw', 'private'], } ) @@ -1235,13 +1246,13 @@ def _mounts(self, uid, gid): 'source': f'/etc/netns/{self.netns.network_ns_name}/resolv.conf', 'destination': '/etc/resolv.conf', 'type': 'none', - 'options': ['rbind', 'ro'], + 'options': ['bind', 'ro', 'private'], }, { 'source': f'/etc/netns/{self.netns.network_ns_name}/hosts', 'destination': '/etc/hosts', 'type': 'none', - 'options': ['rbind', 'ro'], + 'options': ['bind', 'ro', 'private'], }, ] ) @@ -1484,7 +1495,7 @@ def __init__( 'source': self.io_host_path(), 'destination': '/io', 'type': 'none', - 'options': ['rbind', 'rw'], + 'options': ['bind', 'rw', 'private'], } self.input_volume_mounts.append(io_volume_mount) self.main_volume_mounts.append(io_volume_mount) @@ -1642,7 +1653,7 @@ def __init__( 'source': f'{self.cloudfuse_data_path(bucket)}', 'destination': config['mount_path'], 'type': 'none', - 'options': ['rbind', 'rw', 'shared'], + 'options': ['bind', 'rw', 'private'], } ) @@ -1652,7 +1663,7 @@ def __init__( 'source': self.secret_host_path(secret), 'destination': secret["mount_path"], 'type': 'none', - 'options': ['rbind', 'rw'], + 'options': ['bind', 'rw', 'private'], } self.main_volume_mounts.append(volume_mount) # this will be the user credentials @@ -2348,37 +2359,37 @@ async def create_and_start( 'source': JVM.SPARK_HOME, 'destination': JVM.SPARK_HOME, 'type': 'none', - 'options': ['rbind', 'rw'], + 'options': ['bind', 'rw', 'private'], }, { 'source': '/jvm-entryway', 'destination': '/jvm-entryway', 'type': 'none', - 'options': ['rbind', 'rw'], + 'options': ['bind', 'rw', 'private'], }, { 'source': '/hail-jars', 'destination': '/hail-jars', 'type': 'none', - 'options': ['rbind', 'rw'], + 'options': ['bind', 'rw', 'private'], }, { 'source': root_dir, 'destination': root_dir, 'type': 'none', - 'options': ['rbind', 'rw'], + 'options': ['bind', 'rw', 'private'], }, { 'source': '/batch', 'destination': '/batch', 'type': 'none', - 'options': ['rbind', 'rw'], + 'options': ['bind', 'rw', 'private'], }, { 'source': cloudfuse_dir, 'destination': '/cloudfuse', 'type': 'none', - 'options': ['rbind', 'ro', 'rslave'], + 'options': ['rbind', 'ro', 'slave'], }, ] From 36fae4ae48422509e7f0a28a23bf52d239939deb Mon Sep 17 00:00:00 2001 From: Dan King Date: Wed, 3 May 2023 03:48:33 -0400 Subject: [PATCH 17/26] [gcr] eliminate GCR (#12963) --- batch/test/test_batch.py | 6 +++--- benchmark/Makefile | 2 +- dev-docs/development_process.md | 2 +- hail/Makefile | 8 ++++---- hail/python/hailtop/batch/docs/cookbook/clumping.rst | 6 +++--- .../batch/docs/cookbook/files/batch_clumping.py | 2 +- .../hailtop/batch/docs/cookbook/random_forest.rst | 6 +++--- hail/python/hailtop/batch/docs/docker_resources.rst | 12 ++++++------ infra/gcp/README.md | 7 ++++--- infra/gcp/bootstrap.sh | 7 ++++--- 10 files changed, 30 insertions(+), 28 deletions(-) diff --git a/batch/test/test_batch.py b/batch/test/test_batch.py index 3c3b5783cf2..1d0a3515872 100644 --- a/batch/test/test_batch.py +++ b/batch/test/test_batch.py @@ -362,15 +362,15 @@ def test_unknown_image(client: BatchClient): @skip_in_azure -def test_invalid_gcr(client: BatchClient): +def test_invalid_gar(client: BatchClient): bb = create_batch(client) # GCP projects can't be strictly numeric - j = bb.create_job('gcr.io/1/does-not-exist', ['echo', 'test']) + j = bb.create_job('us-docker.pkg.dev/1/does-not-exist', ['echo', 'test']) b = bb.submit() status = j.wait() try: assert j._get_exit_code(status, 'main') is None - assert status['status']['container_statuses']['main']['short_error'] == 'image repository is invalid', str( + assert status['status']['container_statuses']['main']['short_error'] == 'image cannot be pulled', str( (status, b.debug_info()) ) except Exception as e: diff --git a/benchmark/Makefile b/benchmark/Makefile index 5f6d7dba674..0891843d0e5 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -30,7 +30,7 @@ cleanup_image: BENCHMARK_PROJECT ?= hail-vdc BENCHMARK_DOCKER_TAG ?= benchmark_$(shell whoami) -BENCHMARK_REPO_BASE = gcr.io/$(BENCHMARK_PROJECT)/$(BENCHMARK_DOCKER_TAG) +BENCHMARK_REPO_BASE = us-docker.pkg.dev/$(BENCHMARK_PROJECT)/$(BENCHMARK_DOCKER_TAG) DOCKER_ROOT_IMAGE := ubuntu:20.04 ifndef HAIL_WHEEL diff --git a/dev-docs/development_process.md b/dev-docs/development_process.md index c730c72eff2..046d8646a7a 100644 --- a/dev-docs/development_process.md +++ b/dev-docs/development_process.md @@ -66,7 +66,7 @@ Install and configure tools necessary for working on the Hail Services: gcloud auth login gcloud config set project hail-vdc gcloud container clusters get-credentials vdc --zone=us-central1-a -gcloud auth -q configure-docker gcr.io +gcloud auth -q configure-docker us-docker.pkg.dev ``` 5. Add these lines to `~/.zshrc` or `~/.bashrc` to configure your shell and environment for Hail: diff --git a/hail/Makefile b/hail/Makefile index c78c20bc6a4..edd0bf95166 100644 --- a/hail/Makefile +++ b/hail/Makefile @@ -367,16 +367,16 @@ test-dataproc-38: install-hailctl # set GITHUB_OAUTH_HEADER_FILE to that filename # # create the hailgenetics/hail image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_HAIL_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/hail:1234abcd +# set HAIL_GENETICS_HAIL_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/hail:1234abcd # # create the hailgenetics/hailtop image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_HAILTOP_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/hailtop:1234abcd +# set HAIL_GENETICS_HAILTOP_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/hailtop:1234abcd # # create the hailgenetics/vep-grch37-85 image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_VEP_GRCH37_85_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/vep-grch37-85:1234abcd +# set HAIL_GENETICS_VEP_GRCH37_85_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/vep-grch37-85:1234abcd # # create the hailgenetics/vep-grch38-95 image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_VEP_GRCH38_95_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/vep-grch38-95:1234abcd +# set HAIL_GENETICS_VEP_GRCH38_95_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/vep-grch38-95:1234abcd # # build a Azure-HDInsight-compatible wheel file (see build.yaml "build_wheel_for_azure" or start a # cluster to find the correct Scala and Spark versions because the version webpage does not include diff --git a/hail/python/hailtop/batch/docs/cookbook/clumping.rst b/hail/python/hailtop/batch/docs/cookbook/clumping.rst index 902ff689198..db34d54d11d 100644 --- a/hail/python/hailtop/batch/docs/cookbook/clumping.rst +++ b/hail/python/hailtop/batch/docs/cookbook/clumping.rst @@ -96,8 +96,8 @@ The following Docker command pushes the image to GCR: .. code-block:: sh - docker tag 1kg-gwas gcr.io//1kg-gwas - docker push gcr.io//1kg-gwas + docker tag 1kg-gwas us-docker.pkg.dev//1kg-gwas + docker push us-docker.pkg.dev//1kg-gwas Replace ```` with the name of your Google project. Ensure your Batch service account :ref:`can access images in GCR `. @@ -129,7 +129,7 @@ access the binary PLINK file output and association results in downstream jobs. """ cores = 2 g = batch.new_job(name='run-gwas') - g.image('gcr.io//1kg-gwas:latest') + g.image('us-docker.pkg.dev//1kg-gwas:latest') g.cpu(cores) g.declare_resource_group(ofile={ 'bed': '{root}.bed', diff --git a/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py b/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py index d0e539b815d..f3e0bb4eddf 100644 --- a/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py +++ b/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py @@ -7,7 +7,7 @@ def gwas(batch, vcf, phenotypes): """ cores = 2 g = batch.new_job(name='run-gwas') - g.image('gcr.io//1kg-gwas:latest') + g.image('us-docker.pkg.dev//1kg-gwas:latest') g.cpu(cores) g.declare_resource_group(ofile={ 'bed': '{root}.bed', diff --git a/hail/python/hailtop/batch/docs/cookbook/random_forest.rst b/hail/python/hailtop/batch/docs/cookbook/random_forest.rst index 0fdb2a391e1..badc7e80775 100644 --- a/hail/python/hailtop/batch/docs/cookbook/random_forest.rst +++ b/hail/python/hailtop/batch/docs/cookbook/random_forest.rst @@ -129,12 +129,12 @@ along with any desired Python packages. For running the random forest, we need both the `sklearn` and `pandas` Python packages installed in the image. We use :func:`.docker.build_python_image` to build -an image and push it automatically to the location specified (ex: `gcr.io/hail-vdc/random-forest`). +an image and push it automatically to the location specified (ex: `us-docker.pkg.dev/hail-vdc/random-forest`). .. code-block:: python - image = hb.build_python_image('gcr.io/hail-vdc/random-forest', - requirements=['sklearn', 'pandas']) + image = hb.build_python_image('us-docker.pkg.dev/hail-vdc/random-forest', + requirements=['sklearn', 'pandas']) ~~~~~~~~~~~~ Control Code diff --git a/hail/python/hailtop/batch/docs/docker_resources.rst b/hail/python/hailtop/batch/docs/docker_resources.rst index 9b3b9d28c2d..a7573aed938 100644 --- a/hail/python/hailtop/batch/docs/docker_resources.rst +++ b/hail/python/hailtop/batch/docs/docker_resources.rst @@ -75,21 +75,21 @@ To create a Docker image, use .. code-block:: sh - docker build -t gcr.io//: -f Dockerfile . + docker build -t us-docker.pkg.dev//: -f Dockerfile . * `` is the context directory, `.` means the current working directory, * `-t ` specifies the image name, and * `-f ` specifies the Dockerfile file. * A more complete description may be found `here: `__. -For example, we can build an image named gcr.io// based on the Dockerfile named Dockerfile, using the current working directory as the context: +For example, we can build an image named us-docker.pkg.dev// based on the Dockerfile named Dockerfile, using the current working directory as the context: .. code-block:: sh - docker build -t gcr.io//: -f Dockerfile . + docker build -t us-docker.pkg.dev//: -f Dockerfile . -In this example we prepend the image name with `gcr.io//` so that it may be pushed to the Google Container Registry, in the next step. +In this example we prepend the image name with `us-docker.pkg.dev//` so that it may be pushed to the Google Container Registry, in the next step. Pushing Images -------------- @@ -100,8 +100,8 @@ Docker Hub. Below is an example of pushing the image to the Google Container Reg .. code-block:: sh - docker push gcr.io//: + docker push us-docker.pkg.dev//: Now you can use your Docker image with Batch to run your code with the method :meth:`.BashJob.image` -specifying the image as `gcr.io//:`! +specifying the image as `us-docker.pkg.dev//:`! diff --git a/infra/gcp/README.md b/infra/gcp/README.md index 4f8c71d0c1d..fb17d2ee635 100644 --- a/infra/gcp/README.md +++ b/infra/gcp/README.md @@ -208,11 +208,12 @@ You can now install Hail: for Docker can be applied). The following steps should be completed from the $HAIL/infra/gcp directory, unless otherwise stated. -- Run the following to authenticate docker and kubectl with the new - container registry and kubernetes cluster, respectively. +- Run the following to authenticate docker and kubectl with the new artifact + registry and kubernetes cluster, respectively. The `GKE_ZONE` is the zone of + the GKE cluster and the `GAR_REGION` is the region of the artifact registry. ``` - ./bootstrap.sh configure_gcloud + ./bootstrap.sh configure_gcloud ``` - Edit `$HAIL/letsencrypt/subdomains.txt` to include just the services you plan diff --git a/infra/gcp/bootstrap.sh b/infra/gcp/bootstrap.sh index 87009ec73d8..85ffd30946d 100755 --- a/infra/gcp/bootstrap.sh +++ b/infra/gcp/bootstrap.sh @@ -4,10 +4,11 @@ source ../bootstrap_utils.sh function configure_gcloud() { ZONE=${1:-"us-central1-a"} + REGION=${2:-"us"} - gcloud -q auth configure-docker - # If you are using the Artifact Registry: - # gcloud -q auth configure-docker $REGION-docker.pkg.dev + # If you are using the Container Registry: + # gcloud -q auth configure-docker + gcloud -q auth configure-docker $REGION-docker.pkg.dev gcloud container clusters get-credentials --zone $ZONE vdc } From bcd5bb5862721ee6078114d9cfff3592e6fce7e8 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Wed, 3 May 2023 09:52:50 -0400 Subject: [PATCH 18/26] [docs] Clarify that developer dependencies require the JDK not just the JRE (#12968) --- hail/python/hail/docs/getting_started_developing.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hail/python/hail/docs/getting_started_developing.rst b/hail/python/hail/docs/getting_started_developing.rst index a1fb22467a4..23a3df0bfdd 100644 --- a/hail/python/hail/docs/getting_started_developing.rst +++ b/hail/python/hail/docs/getting_started_developing.rst @@ -6,11 +6,13 @@ Hail is an open-source project. We welcome contributions to the repository. Requirements ~~~~~~~~~~~~ -- `Java 8 or 11 JDK `_ +- `Java 8 or 11 JDK `_. Note: it *must* be Java **8** or Java **11**. Hail does not support versions 9-10 or 12+ due to our dependency on Spark. -- The Python and non-pip installation requirements in `Getting Started `_ +- The Python and non-pip installation requirements in `Getting Started `_. + Note: These instructions install the JRE but that is not necessary as the JDK should already + be installed which includes the JRE. - If you are setting `HAIL_COMPILE_NATIVES=1`, then you need the LZ4 library header files. On Debian and Ubuntu machines run: `apt-get install liblz4-dev`. From f81cec8f79c3afde2816720890b8b210a6ad4d11 Mon Sep 17 00:00:00 2001 From: Dan King Date: Wed, 3 May 2023 11:03:38 -0400 Subject: [PATCH 19/26] [batch] use regional bucket for requester pays tests (#12964) I updated terraform but 1. GCP Terraform state is still local on my laptop. 2. GCP Terraform appears to not configure global-config. As such, I cannot thread the name of the bucket through to the tests the way we do with TEST_STORAGE_URI. For now, I've hardcoded the name (which is what we were doing previously). When we eventually get to testing recreation of GCP in a new project we'll have to address the global config then. --- .../test/hail/fs/test_worker_driver_fs.py | 34 +++++++++---------- hail/python/test/hailtop/batch/test_batch.py | 4 +-- infra/gcp-broad/main.tf | 15 ++++++++ 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/hail/python/test/hail/fs/test_worker_driver_fs.py b/hail/python/test/hail/fs/test_worker_driver_fs.py index 344c670a7eb..d8100a6b3db 100644 --- a/hail/python/test/hail/fs/test_worker_driver_fs.py +++ b/hail/python/test/hail/fs/test_worker_driver_fs.py @@ -8,7 +8,7 @@ @skip_in_azure def test_requester_pays_no_settings(): try: - hl.import_table('gs://hail-services-requester-pays/hello') + hl.import_table('gs://hail-test-requester-pays-fds32/hello') except Exception as exc: assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0] else: @@ -17,7 +17,7 @@ def test_requester_pays_no_settings(): @skip_in_azure def test_requester_pays_write_no_settings(): - random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) + random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) try: hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True) except Exception as exc: @@ -32,7 +32,7 @@ def test_requester_pays_write_no_settings(): def test_requester_pays_write_with_project(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - random_filename = 'gs://hail-services-requester-pays/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) + random_filename = 'gs://hail-test-requester-pays-fds32/test_requester_pays_on_worker_driver_' + secret_alnum_string(10) try: hl.utils.range_table(4, n_partitions=4).write(random_filename, overwrite=True) finally: @@ -43,20 +43,20 @@ def test_requester_pays_write_with_project(): def test_requester_pays_with_project(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays'])) - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket'])) - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] hl.stop() hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket'])) try: - hl.import_table('gs://hail-services-requester-pays/hello') + hl.import_table('gs://hail-test-requester-pays-fds32/hello') except Exception as exc: assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0] else: @@ -64,7 +64,7 @@ def test_requester_pays_with_project(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/hello', no_header=True).collect() == [hl.Struct(f0='hello')] + assert hl.import_table('gs://hail-test-requester-pays-fds32/hello', no_header=True).collect() == [hl.Struct(f0='hello')] @skip_in_azure @@ -93,20 +93,20 @@ def test_requester_pays_with_project_more_than_one_partition(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays'])) - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents hl.stop() - hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-services-requester-pays', 'other-bucket'])) - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + hl.init(gcs_requester_pays_configuration=('hail-vdc', ['hail-test-requester-pays-fds32', 'other-bucket'])) + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents hl.stop() hl.init(gcs_requester_pays_configuration=('hail-vdc', ['other-bucket'])) try: - hl.import_table('gs://hail-services-requester-pays/zero-to-nine', min_partitions=8) + hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', min_partitions=8) except Exception as exc: assert "Bucket is a requester pays bucket but no user project provided" in exc.args[0] else: @@ -114,7 +114,7 @@ def test_requester_pays_with_project_more_than_one_partition(): hl.stop() hl.init(gcs_requester_pays_configuration='hail-vdc') - assert hl.import_table('gs://hail-services-requester-pays/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents + assert hl.import_table('gs://hail-test-requester-pays-fds32/zero-to-nine', no_header=True, min_partitions=8).collect() == expected_file_contents @run_if_azure diff --git a/hail/python/test/hailtop/batch/test_batch.py b/hail/python/test/hailtop/batch/test_batch.py index d94bc910956..e4ea03eada6 100644 --- a/hail/python/test/hailtop/batch/test_batch.py +++ b/hail/python/test/hailtop/batch/test_batch.py @@ -748,7 +748,7 @@ def test_cloudfuse_empty_string_bucket_fails(self): def test_fuse_requester_pays(self): b = self.batch(requester_pays_project='hail-vdc') j = b.new_job() - j.cloudfuse('hail-services-requester-pays', '/fuse-bucket') + j.cloudfuse('hail-test-requester-pays-fds32', '/fuse-bucket') j.command('cat /fuse-bucket/hello') res = b.run() res_status = res.status() @@ -776,7 +776,7 @@ def test_fuse_non_requester_pays_bucket_when_requester_pays_project_specified(se @skip_in_azure def test_requester_pays(self): b = self.batch(requester_pays_project='hail-vdc') - input = b.read_input('gs://hail-services-requester-pays/hello') + input = b.read_input('gs://hail-test-requester-pays-fds32/hello') j = b.new_job() j.command(f'cat {input}') res = b.run() diff --git a/infra/gcp-broad/main.tf b/infra/gcp-broad/main.tf index b208cfe5896..d198e8d149f 100644 --- a/infra/gcp-broad/main.tf +++ b/infra/gcp-broad/main.tf @@ -628,6 +628,21 @@ resource "google_storage_bucket" "hail_test_bucket" { timeouts {} } +resource "random_string" "hail_test_requester_pays_bucket_suffix" { + length = 5 +} + +resource "google_storage_bucket" "hail_test_requester_pays_bucket" { + name = "hail-test-requester-pays-${random_string.hail_test_requester_pays_bucket_suffix.result}" + location = var.hail_test_gcs_bucket_location + force_destroy = false + storage_class = var.hail_test_gcs_bucket_storage_class + uniform_bucket_level_access = true + requester_pays = true + + timeouts {} +} + resource "google_dns_managed_zone" "dns_zone" { description = "" name = "hail" From f6017673dbb6589247b0b2b4b11483e56a37807a Mon Sep 17 00:00:00 2001 From: jigold Date: Wed, 3 May 2023 13:03:58 -0400 Subject: [PATCH 20/26] [hailtop.batch] Fix 12924 -- waiting on a batch with zero unsubmitted jobs (#12953) Fixes #12924 --- hail/python/hailtop/batch/backend.py | 2 +- hail/python/test/hailtop/batch/test_batch.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hail/python/hailtop/batch/backend.py b/hail/python/hailtop/batch/backend.py index 4c96ce547d3..64b000dfe64 100644 --- a/hail/python/hailtop/batch/backend.py +++ b/hail/python/hailtop/batch/backend.py @@ -786,7 +786,7 @@ async def compile_job(job): if open: webbrowser.open(url) - if wait: + if wait and len(unsubmitted_jobs) > 0: if verbose: print(f'Waiting for batch {batch_handle.id}...') starting_job_id = min(j._client_job.job_id for j in unsubmitted_jobs) diff --git a/hail/python/test/hailtop/batch/test_batch.py b/hail/python/test/hailtop/batch/test_batch.py index e4ea03eada6..8202594a798 100644 --- a/hail/python/test/hailtop/batch/test_batch.py +++ b/hail/python/test/hailtop/batch/test_batch.py @@ -1300,3 +1300,8 @@ def write(kwargs): res_status = res.status() assert res_status['state'] == 'success', str((res_status, res.debug_info())) assert res.get_job_log(tail._job_id)['main'] == 'ab', str(res.debug_info()) + + def test_wait_on_empty_batch_update(self): + b = self.batch() + b.run(wait=True) + b.run(wait=True) From c1b6a62a3ca993f0d58fea78c22651f1625b81c4 Mon Sep 17 00:00:00 2001 From: Dan King Date: Wed, 3 May 2023 14:10:33 -0400 Subject: [PATCH 21/26] [batch] increment worker instance version (#12955) https://github.com/hail-is/hail/commit/1940547d35ddddb084ad52684e36153c1e03a331 should have incremented the worker version. New QoB logging is incompatible with old workers. --- batch/batch/globals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/batch/batch/globals.py b/batch/batch/globals.py index d232d79401a..cbf6e3326da 100644 --- a/batch/batch/globals.py +++ b/batch/batch/globals.py @@ -21,7 +21,7 @@ BATCH_FORMAT_VERSION = 7 STATUS_FORMAT_VERSION = 5 -INSTANCE_VERSION = 23 +INSTANCE_VERSION = 24 MAX_PERSISTENT_SSD_SIZE_GIB = 64 * 1024 RESERVED_STORAGE_GB_PER_CORE = 5 From 330031a5d9734fd33a50e5651e7a2505f352b239 Mon Sep 17 00:00:00 2001 From: jigold Date: Wed, 3 May 2023 16:55:53 -0400 Subject: [PATCH 22/26] [batch] Ensure submount in /io doesn't cause deletion (#12977) This PR just adds a test to make sure submounts don't cause deletion. --- hail/python/test/hailtop/batch/test_batch.py | 39 +++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/hail/python/test/hailtop/batch/test_batch.py b/hail/python/test/hailtop/batch/test_batch.py index 8202594a798..b7674754d4d 100644 --- a/hail/python/test/hailtop/batch/test_batch.py +++ b/hail/python/test/hailtop/batch/test_batch.py @@ -504,25 +504,25 @@ def setUp(self): if not os.path.exists(in_cluster_key_file): in_cluster_key_file = None - router_fs = RouterAsyncFS(gcs_kwargs={'gcs_requester_pays_configuration': 'hail-vdc', 'credentials_file': in_cluster_key_file}, - azure_kwargs={'credential_file': in_cluster_key_file}) + self.router_fs = RouterAsyncFS(gcs_kwargs={'gcs_requester_pays_configuration': 'hail-vdc', 'credentials_file': in_cluster_key_file}, + azure_kwargs={'credential_file': in_cluster_key_file}) - def sync_exists(url): - return async_to_blocking(router_fs.exists(url)) - - def sync_write(url, data): - return async_to_blocking(router_fs.write(url, data)) - - if not sync_exists(f'{self.remote_tmpdir}batch-tests/resources/hello.txt'): - sync_write(f'{self.remote_tmpdir}batch-tests/resources/hello.txt', b'hello world') - if not sync_exists(f'{self.remote_tmpdir}batch-tests/resources/hello spaces.txt'): - sync_write(f'{self.remote_tmpdir}batch-tests/resources/hello spaces.txt', b'hello') - if not sync_exists(f'{self.remote_tmpdir}batch-tests/resources/hello (foo) spaces.txt'): - sync_write(f'{self.remote_tmpdir}batch-tests/resources/hello (foo) spaces.txt', b'hello') + if not self.sync_exists(f'{self.remote_tmpdir}batch-tests/resources/hello.txt'): + self.sync_write(f'{self.remote_tmpdir}batch-tests/resources/hello.txt', b'hello world') + if not self.sync_exists(f'{self.remote_tmpdir}batch-tests/resources/hello spaces.txt'): + self.sync_write(f'{self.remote_tmpdir}batch-tests/resources/hello spaces.txt', b'hello') + if not self.sync_exists(f'{self.remote_tmpdir}batch-tests/resources/hello (foo) spaces.txt'): + self.sync_write(f'{self.remote_tmpdir}batch-tests/resources/hello (foo) spaces.txt', b'hello') def tearDown(self): self.backend.close() + def sync_exists(self, url): + return async_to_blocking(self.router_fs.exists(url)) + + def sync_write(self, url, data): + return async_to_blocking(self.router_fs.write(url, data)) + def batch(self, requester_pays_project=None, default_python_image=None, cancel_after_n_failures=None): name_of_test_method = inspect.stack()[1][3] @@ -744,6 +744,17 @@ def test_cloudfuse_empty_string_bucket_fails(self): with self.assertRaises(BatchException): j.cloudfuse(self.bucket, '') + def test_cloudfuse_submount_in_io_doesnt_rm_bucket(self): + assert self.bucket + b = self.batch() + j = b.new_job() + j.cloudfuse(self.bucket, '/io/cloudfuse') + j.command(f'ls /io/cloudfuse/') + res = b.run() + res_status = res.status() + assert res_status['state'] == 'success', str((res_status, res.debug_info())) + assert self.sync_exists(f'{self.remote_tmpdir}batch-tests/resources/hello.txt') + @skip_in_azure def test_fuse_requester_pays(self): b = self.batch(requester_pays_project='hail-vdc') From 6d43c7e26a3c779fecdb373e785a477434c6f57e Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Wed, 3 May 2023 20:22:16 -0400 Subject: [PATCH 23/26] [ci] Test pip-installed lints against python 3.9 (#12970) Because the python version in hail-ubuntu changed from 3.7 to 3.8, both of the `hail-pip-installed` dockerfiles were running 3.8. I changed the old 3.7 dockerfile to try to use 3.9 --- build.yaml | 16 ++++++++-------- hail/Dockerfile.hail-pip-installed-python38 | 5 +---- ...37 => Dockerfile.hail-pip-installed-python39} | 5 ++++- 3 files changed, 13 insertions(+), 13 deletions(-) rename hail/{Dockerfile.hail-pip-installed-python37 => Dockerfile.hail-pip-installed-python39} (75%) diff --git a/build.yaml b/build.yaml index 3c5b835216a..d81d31dec4b 100644 --- a/build.yaml +++ b/build.yaml @@ -1002,13 +1002,13 @@ steps: - build_hail - merge_code - kind: buildImage2 - name: hail_pip_installed_python37_image - dockerFile: /io/repo/hail/Dockerfile.hail-pip-installed-python37 + name: hail_pip_installed_python39_image + dockerFile: /io/repo/hail/Dockerfile.hail-pip-installed-python39 contextPath: /io/repo - publishAs: hail-pip-installed-python37 + publishAs: hail-pip-installed-python39 inputs: - - from: /repo/hail/Dockerfile.hail-pip-installed-python37 - to: /io/repo/hail/Dockerfile.hail-pip-installed-python37 + - from: /repo/hail/Dockerfile.hail-pip-installed-python39 + to: /io/repo/hail/Dockerfile.hail-pip-installed-python39 - from: /repo/hail/python/pinned-requirements.txt to: /io/repo/hail/python/pinned-requirements.txt - from: /repo/hail/python/dev/pinned-requirements.txt @@ -1046,9 +1046,9 @@ steps: - hail_ubuntu_image - merge_code - kind: runImage - name: check_hail_python37 + name: check_hail_python39 image: - valueFrom: hail_pip_installed_python37_image.image + valueFrom: hail_pip_installed_python39_image.image script: | set -x SITE_PACKAGES=$(pip3 show hail | grep Location | sed 's/Location: //') @@ -1063,7 +1063,7 @@ steps: exit $exit_status dependsOn: - - hail_pip_installed_python37_image + - hail_pip_installed_python39_image - kind: runImage name: check_hail_python38 image: diff --git a/hail/Dockerfile.hail-pip-installed-python38 b/hail/Dockerfile.hail-pip-installed-python38 index 17ed6ca592a..23354c44a46 100644 --- a/hail/Dockerfile.hail-pip-installed-python38 +++ b/hail/Dockerfile.hail-pip-installed-python38 @@ -1,10 +1,7 @@ FROM {{ hail_ubuntu_image.image }} ENV LANG C.UTF-8 -RUN hail-apt-get-install \ - openjdk-8-jdk-headless \ - python3.8 python3-pip \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 +RUN hail-apt-get-install openjdk-8-jdk-headless COPY hail/python/pinned-requirements.txt requirements.txt COPY hail/python/dev/pinned-requirements.txt dev-requirements.txt diff --git a/hail/Dockerfile.hail-pip-installed-python37 b/hail/Dockerfile.hail-pip-installed-python39 similarity index 75% rename from hail/Dockerfile.hail-pip-installed-python37 rename to hail/Dockerfile.hail-pip-installed-python39 index 25ab7a6f4b2..85ac5b30949 100644 --- a/hail/Dockerfile.hail-pip-installed-python37 +++ b/hail/Dockerfile.hail-pip-installed-python39 @@ -2,7 +2,10 @@ FROM {{ hail_ubuntu_image.image }} ENV LANG C.UTF-8 -RUN hail-apt-get-install openjdk-8-jdk-headless +RUN hail-apt-get-install \ + openjdk-8-jdk-headless \ + python3.9 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 COPY hail/python/pinned-requirements.txt requirements.txt COPY hail/python/dev/pinned-requirements.txt dev-requirements.txt From be52d40784cd7ed194ac5e07417e5b4b089bdb7e Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Wed, 3 May 2023 21:59:27 -0400 Subject: [PATCH 24/26] [batch] Skip currently incorrect tests after Azure price change (#12979) Skipping these for the timebeing because they currently only pass for 16-core worker VMs and not for 8-core worker VMs. --- batch/test/test_batch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/batch/test/test_batch.py b/batch/test/test_batch.py index 1d0a3515872..a1486cfd8b1 100644 --- a/batch/test/test_batch.py +++ b/batch/test/test_batch.py @@ -11,7 +11,7 @@ from hailtop.batch.backend import HAIL_GENETICS_HAILTOP_IMAGE from hailtop.batch_client.client import BatchClient from hailtop.config import get_deploy_config, get_user_config -from hailtop.test_utils import fails_in_azure, skip_in_azure +from hailtop.test_utils import skip_in_azure from hailtop.utils import external_requests_client_session, retry_response_returning_functions, sync_sleep_and_backoff from .failure_injecting_client_session import FailureInjectingClientSession @@ -145,7 +145,7 @@ def test_invalid_resource_requests(client: BatchClient): bb.submit() -@fails_in_azure # https://github.com/hail-is/hail/issues/12958 +@skip_in_azure # https://github.com/hail-is/hail/issues/12958 def test_out_of_memory(client: BatchClient): bb = create_batch(client) resources = {'cpu': '0.25', 'memory': '10M', 'storage': '10Gi'} @@ -1008,7 +1008,7 @@ def test_pool_highcpu_instance(client: BatchClient): assert 'highcpu' in status['status']['worker'], str((status, b.debug_info())) -@fails_in_azure # https://github.com/hail-is/hail/issues/12958 +@skip_in_azure # https://github.com/hail-is/hail/issues/12958 def test_pool_highcpu_instance_cheapest(client: BatchClient): bb = create_batch(client) resources = {'cpu': '0.25', 'memory': '50Mi'} From 78ee77e38f51055e90360777b6e015f92b2afa33 Mon Sep 17 00:00:00 2001 From: jigold Date: Thu, 4 May 2023 15:59:31 -0400 Subject: [PATCH 25/26] [batch] Don't rmtree if any errors occur while unmounting (#12985) I think this change is better than #12975 --- batch/batch/worker/worker.py | 39 ++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/batch/batch/worker/worker.py b/batch/batch/worker/worker.py index 80d03d03d1f..ce7dcbb4c35 100644 --- a/batch/batch/worker/worker.py +++ b/batch/batch/worker/worker.py @@ -1936,6 +1936,7 @@ async def cleanup(self): log.exception( f'while unmounting fuse blob storage {bucket} from {mount_path} for job {self.id}' ) + raise try: async with async_timeout.timeout(120): @@ -2212,25 +2213,32 @@ async def cleanup(self): assert self.worker.fs assert self.jvm + with self.step('uploading_log'): + log_contents = await self.worker.fs.read(self.log_file) + await self.worker.file_store.write_log_file( + self.format_version, self.batch_id, self.job_id, self.attempt_id, 'main', log_contents + ) + if self.cloudfuse: for config in self.cloudfuse: if config['mounted']: bucket = config['bucket'] assert bucket mount_path = self.cloudfuse_data_path(bucket) - await self.jvm.cloudfuse_mount_manager.unmount(mount_path, user=self.user, bucket=bucket) - config['mounted'] = False + try: + await self.jvm.cloudfuse_mount_manager.unmount(mount_path, user=self.user, bucket=bucket) + config['mounted'] = False + except asyncio.CancelledError: + raise + except Exception as e: + raise IncompleteJVMCleanupError( + f'while unmounting fuse blob storage {bucket} from {mount_path} for {self.jvm_name} for job {self.id}' + ) from e if self.jvm is not None: self.worker.return_jvm(self.jvm) self.jvm = None - with self.step('uploading_log'): - log_contents = await self.worker.fs.read(self.log_file) - await self.worker.file_store.write_log_file( - self.format_version, self.batch_id, self.job_id, self.attempt_id, 'main', log_contents - ) - try: await check_shell(f'xfs_quota -x -c "limit -p bsoft=0 bhard=0 {self.project_id}" /host') await blocking_to_async(self.pool, shutil.rmtree, self.scratch, ignore_errors=True) @@ -2312,6 +2320,10 @@ class JVMCreationError(Exception): pass +class IncompleteJVMCleanupError(Exception): + pass + + class JVMUserCredentials: pass @@ -2734,6 +2746,12 @@ def return_jvm(self, jvm: JVM): jvm.reset() self._jvms.add(jvm) + async def recreate_jvm(self, jvm: JVM): + self._jvms.remove(jvm) + log.info(f'quarantined {jvm} and recreated a new jvm') + new_jvm = await JVM.create(jvm.index, jvm.n_cores, self) + self._jvms.add(new_jvm) + async def shutdown(self): log.info('Worker.shutdown') self._jvm_initializer_task.cancel() @@ -2771,6 +2789,11 @@ async def run_job(self, job): raise except JVMCreationError: self.stop_event.set() + except IncompleteJVMCleanupError: + assert isinstance(job, JVMJob) + assert job.jvm is not None + await self.recreate_jvm(job.jvm) + log.exception(f'while running {job}, ignoring') except Exception as e: if not user_error(e): log.exception(f'while running {job}, ignoring') From da6ba693ab4473f940a355f7c5de10a035713b3d Mon Sep 17 00:00:00 2001 From: jigold Date: Thu, 4 May 2023 19:06:21 -0400 Subject: [PATCH 26/26] [batch] Check /proc/mounts for straggler cloudfuse mounts (#12986) --- batch/batch/worker/worker.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/batch/batch/worker/worker.py b/batch/batch/worker/worker.py index ce7dcbb4c35..c4488654553 100644 --- a/batch/batch/worker/worker.py +++ b/batch/batch/worker/worker.py @@ -692,6 +692,10 @@ class ContainerStartError(Exception): pass +class IncompleteCloudFuseCleanup(Exception): + pass + + def worker_fraction_in_1024ths(cpu_in_mcpu): return 1024 * cpu_in_mcpu // (CORES * 1000) @@ -1938,6 +1942,11 @@ async def cleanup(self): ) raise + with open('/proc/mounts', 'r', encoding='utf-8') as f: + output = f.read() + if self.cloudfuse_base_path() in output: + raise IncompleteCloudFuseCleanup(f'incomplete cloudfuse unmounting: {output}') + try: async with async_timeout.timeout(120): await check_shell(f'xfs_quota -x -c "limit -p bsoft=0 bhard=0 {self.project_id}" /host')