Uses bind volume instead of docker volume for MSSQL docker in tmpfs (…

…#16159) Seems that MSSQL is not able to use data volume when it is mounted from tmpfs filesystem. See microsoft/mssql-docker#13 In such case, instead of mounting docker-created volume we mount a volume mounted from home directory of the user which is unlikely to be a tmpfs volume. GitOrigin-RevId: 352fefaef1712bf5e60e3e79a86214279be69e16
GoogleCloudPlatform · Jun 4, 2022 · 484f035 · 484f035
1 parent b59b7df
commit 484f035
Show file tree

Hide file tree

Showing 6 changed files with 111 additions and 4 deletions.
diff --git a/breeze b/breeze
@@ -673,6 +673,22 @@ function breeze::prepare_command_files() {
     local remove_sources_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/remove-sources.yml
     local forward_credentials_docker_compose_file=${SCRIPTS_CI_DIR}/docker-compose/forward-credentials.yml
 
+    if [[ ${BACKEND} == "mssql" ]]; then
+        local docker_filesystem
+        docker_filesystem=$(stat "-f" "-c" "%T" /var/lib/docker || echo "unknown")
+        if [[ ${docker_filesystem} == "tmpfs" ]]; then
+            # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+            # O_DIRECT parameter for direct writing to the filesystem
+            # https://github.com/microsoft/mssql-docker/issues/13
+            # so we need to mount an external volume for its db location
+            # specified by MSSQL_DATA_VOLUME
+            backend_docker_compose_file="${backend_docker_compose_file}:${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-bind-volume.yml"
+        else
+            backend_docker_compose_file="${backend_docker_compose_file}:${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-docker-volume.yml"
+        fi
+    fi
+
+
     local compose_ci_file=${main_ci_docker_compose_file}:${backend_docker_compose_file}:${files_docker_compose_file}
     local compose_prod_file=${main_prod_docker_compose_file}:${backend_docker_compose_file}:${files_docker_compose_file}
 
@@ -1422,6 +1438,13 @@ function breeze::parse_arguments() {
                     INTEGRATIONS+=("${INTEGRATION}")
                 fi
             done
+            # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+            # O_DIRECT parameter for direct writing to the filesystem
+            # https://github.com/microsoft/mssql-docker/issues/13
+            # so we need to mount an external volume for its db location
+            # the external db must allow for parallel testing so external volume is mapped
+            # to the data volume. Stop should also clean the volume
+            rm -rf "${MSSQL_DATA_VOLUME:?"MSSQL_DATA_VOLUME should never be empty!"}"/*
             shift
             ;;
         restart)

diff --git a/scripts/ci/docker-compose/backend-mssql-bind-volume.yml b/scripts/ci/docker-compose/backend-mssql-bind-volume.yml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: "2.2"
+services:
+  mssql:
+    volumes:
+      # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+      # O_DIRECT parameter for direct writing to the filesystem
+      # https://github.com/microsoft/mssql-docker/issues/13
+      # so we need to mount an external volume for its db location
+      # the external db must allow for parallel testing so external volume is mapped
+      # to the data volume
+      - ${MSSQL_DATA_VOLUME}:/var/opt/mssql
diff --git a/scripts/ci/docker-compose/backend-mssql-docker-volume.yml b/scripts/ci/docker-compose/backend-mssql-docker-volume.yml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: "2.2"
+services:
+  mssql:
+    volumes:
+      - mssql-db-volume:/var/opt/mssql
diff --git a/scripts/ci/docker-compose/backend-mssql.yml b/scripts/ci/docker-compose/backend-mssql.yml
@@ -31,8 +31,6 @@ services:
     environment:
       - ACCEPT_EULA=Y
       - SA_PASSWORD=Airflow123
-    volumes:
-      - mssql-db-volume:/var/opt/mssql
     healthcheck:
       test: ["CMD", "/opt/mssql-tools/bin/sqlcmd", "-S", "localhost",
              "-U", "sa", "-P", "Airflow123", "-Q", "SELECT 1"]

diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
@@ -41,12 +41,24 @@ function initialization::create_directories() {
     # As well as hashes of the important files, but also we generate build scripts there that are
     # Used to execute the commands for breeze
     export BUILD_CACHE_DIR="${AIRFLOW_SOURCES}/.build"
-    export BUILD_CACHE_DIR
     readonly BUILD_CACHE_DIR
 
+    # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+    # O_DIRECT parameter for direct writing to the filesystem
+    # https://github.com/microsoft/mssql-docker/issues/13
+    # so we need to mount an external volume for its db location
+    # the external db must allow for parallel testing so external volume is mapped
+    # to the data volume
+    export MSSQL_DATA_VOLUME="${BUILD_CACHE_DIR}/tmp_mssql_volume"
+
     # Create those folders above in case they do not exist
     mkdir -p "${BUILD_CACHE_DIR}" >/dev/null
     mkdir -p "${FILES_DIR}" >/dev/null
+    mkdir -p "${MSSQL_DATA_VOLUME}" >/dev/null
+    # MSSQL 2019 runs with non-root user by default so we have to make the volumes world-writeable
+    # This is a bit scary and we could get by making it group-writeable but the group would have
+    # to be set to "root" (GID=0) for the volume to work and this cannot be accomplished without sudo
+    chmod a+rwx "${MSSQL_DATA_VOLUME}"
 
     # By default we are not in CI environment GitHub Actions sets CI to "true"
     export CI="${CI="false"}"

diff --git a/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh b/scripts/ci/testing/ci_run_single_airflow_test_in_docker.sh
@@ -82,6 +82,29 @@ function run_airflow_testing_in_docker() {
     echo
     echo "Semaphore grabbed. Running tests for ${TEST_TYPE}"
     echo
+    local backend_docker_compose=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml")
+    if [[ ${BACKEND} == "mssql" ]]; then
+        local docker_filesystem
+        docker_filesystem=$(stat "-f" "-c" "%T" /var/lib/docker || echo "unknown")
+        if [[ ${docker_filesystem} == "tmpfs" ]]; then
+            # In case of tmpfs backend for docker, mssql fails because TMPFS does not support
+            # O_DIRECT parameter for direct writing to the filesystem
+            # https://github.com/microsoft/mssql-docker/issues/13
+            # so we need to mount an external volume for its db location
+            # the external db must allow for parallel testing so TEST_TYPE
+            # is added to the volume name
+            export MSSQL_DATA_VOLUME="${HOME}/tmp-mssql-volume-${TEST_TYPE}"
+            mkdir -p "${MSSQL_DATA_VOLUME}"
+            # MSSQL 2019 runs with non-root user by default so we have to make the volumes world-writeable
+            # This is a bit scary and we could get by making it group-writeable but the group would have
+            # to be set to "root" (GID=0) for the volume to work and this cannot be accomplished without sudo
+            chmod a+rwx "${MSSQL_DATA_VOLUME}"
+            backend_docker_compose+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-bind-volume.yml")
+        else
+            backend_docker_compose+=("-f" "${SCRIPTS_CI_DIR}/docker-compose/backend-mssql-docker-volume.yml")
+        fi
+    fi
+
     for try_num in {1..5}
     do
         echo
@@ -90,13 +113,14 @@ function run_airflow_testing_in_docker() {
         echo
         echo "Making sure docker-compose is down and remnants removed"
         echo
+
         docker-compose --log-level INFO -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
             --project-name "airflow-${TEST_TYPE}-${BACKEND}" \
             down --remove-orphans \
             --volumes --timeout 10
         docker-compose --log-level INFO \
           -f "${SCRIPTS_CI_DIR}/docker-compose/base.yml" \
-          -f "${SCRIPTS_CI_DIR}/docker-compose/backend-${BACKEND}.yml" \
+          "${backend_docker_compose[@]}" \
           "${INTEGRATIONS[@]}" \
           "${DOCKER_COMPOSE_LOCAL[@]}" \
           --project-name "airflow-${TEST_TYPE}-${BACKEND}" \