[Draft] Python 3.6 Upgrade / AMO Rebase

circle.yml → .circleci/config.yml

@@ -1,11 +1,16 @@
-version: 2.0
+version: 2.1
 
-jobs:
-  build-deploy: # build for the master branch
-    machine: true
-    working_directory: ~/addons-server
+commands:
+  build_and_push_container_image:
+    description: "Builds and pushes a Docker image"
+    parameters:
+      dockerfile:
+        type: string
+        default: "Dockerfile.python2.deploy"
+      image_tag:
+        type: string
+        default: "latest"
     steps:
-      - checkout
       - run: >
           printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n'
           "$CIRCLE_SHA1"
@@ -18,31 +23,47 @@ jobs:
           name: Build docker image and push to repo
           command: |
             docker version
-            docker build -t app:build -f Dockerfile.deploy .
-            docker tag app:build ${DOCKERHUB_REPO}:latest
-            docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASS
-            docker push ${DOCKERHUB_REPO}:latest
-  build-release: # build for releases (tags)
+            docker build -t app:build -f << parameters.dockerfile >> .
+            docker tag app:build "${DOCKERHUB_REPO}":<< parameters.image_tag >>
+            docker login -u "${DOCKERHUB_USER}" -p "${DOCKERHUB_PASS}"
+            docker push "${DOCKERHUB_REPO}":<< parameters.image_tag >>
+jobs:
+  build-py2:
     machine: true
     working_directory: ~/addons-server
     steps:
       - checkout
-      - run: >
-          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n'
-          "$CIRCLE_SHA1"
-          "$CIRCLE_TAG"
-          "$CIRCLE_PROJECT_USERNAME"
-          "$CIRCLE_PROJECT_REPONAME"
-          "$CIRCLE_BUILD_URL"
-          > version.json
-      - run:
-          name: Build docker image and push to repo
-          command: |
-            docker version
-            docker build -t app:build -f Dockerfile.deploy .
-            docker tag app:build ${DOCKERHUB_REPO}:${CIRCLE_TAG}
-            docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASS
-            docker push ${DOCKERHUB_REPO}:${CIRCLE_TAG}
+      - build_and_push_container_image:
+          image_tag: latest
+          dockerfile: "Dockerfile.python2.deploy"
+
+  build-py3:
+    machine: true
+    working_directory: ~/addons-server
+    steps:
+      - checkout
+      - build_and_push_container_image:
+          image_tag: latest-py3
+          dockerfile: "Dockerfile.python3.deploy"
+
+  build-py2-tag:
+    machine: true
+    working_directory: ~/addons-server
+    steps:
+      - checkout
+      - build_and_push_container_image:
+          image_tag: "${CIRCLE_TAG}"
+          dockerfile: "Dockerfile.python2.deploy"
+
+  build-py3-tag:
+    machine: true
+    working_directory: ~/addons-server
+    steps:
+      - checkout
+      - build_and_push_container_image:
+          image_tag: "${CIRCLE_TAG}-py3"
+          dockerfile: "Dockerfile.python3.deploy"
+
   integration_test:
     working_directory: ~/addons-server
     machine: true
@@ -77,6 +98,7 @@ jobs:
             - uitest-cache-{{ checksum "requirements/prod_py2.txt" }}
             - uitest-cache-{{ checksum "requirements/tests.txt" }}
             - uitest-cache-{{ checksum "requirements/dev.txt" }}
+            - uitest-cache-{{ checksum "requirements/dev_without_hash.txt" }}
             - uitest-cache-{{ checksum "requirements/uitests.txt" }}
             - uitest-cache-
       - run:
@@ -93,23 +115,24 @@ jobs:
           command: |
             set -x
             sudo sysctl -w vm.max_map_count=262144
-            docker-compose pull --quiet
-            docker-compose up -d
+            docker-compose -f docker-compose.yml -f tests/ui/docker-compose.selenium.yml pull --quiet
+            docker-compose -f docker-compose.yml -f tests/ui/docker-compose.selenium.yml up -d
             sleep 20
             docker-compose ps
             scripts/ui-test-hostname-setup.sh
             # Make sure dependencies get updated in worker and web container
-            docker-compose exec worker make -f Makefile-docker update_deps
+            docker-compose exec worker make -f Makefile-docker update_deps update_assets
             docker-compose restart worker
-            docker-compose exec web make -f Makefile-docker update_deps
+            docker-compose exec web make -f Makefile-docker update_deps update_assets
             docker-compose restart web
-            # chown to change permissions of olympia.egg* and all folders/file
-            # This is done because cirlceci has permissions do not allow the user `circleci` to remove or add files create by the root, even if it is part of the root group.
-            sudo chown -R circleci:circleci .
             # Delete any existing uitest users within restmail. TEMP FIX: https://github.com/mozilla/addons-server/issues/8980
             wget --method=DELETE http://restmail.net/mail/uitest
+            # mod user in firefox container
+            docker-compose -f docker-compose.yml -f tests/ui/docker-compose.selenium.yml exec selenium-firefox sudo usermod -u 1001 seluser
+            # proper chown
+            sudo chown -R $USER .
             # Start Test in Firefox docker container
-            docker-compose exec --user circleci selenium-firefox tox -e ui-tests
+            docker-compose -f docker-compose.yml -f tests/ui/docker-compose.selenium.yml exec selenium-firefox tox -e ui-tests
       - store_artifacts:
           path: ui-test.html
       - save_cache:
@@ -128,6 +151,10 @@ jobs:
           key: uitest-cache-{{ checksum "requirements/dev.txt" }}
           paths:
             - .tox
+      - save_cache:
+          key: uitest-cache-{{ checksum "requirements/dev_without_hash.txt" }}
+          paths:
+            - .tox
       - save_cache:
           key: uitest-cache-{{ checksum "requirements/uitests.txt" }}
           paths:
@@ -141,11 +168,21 @@ workflows:
   version: 2
   build_test_deploy_release:
     jobs:
-      - build-deploy:
+      - build-py2:
+          filters:
+            branches:
+              only: master
+      - build-py3:
           filters:
             branches:
               only: master
-      - build-release:
+      - build-py2-tag:
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              ignore: /.*/
+      - build-py3-tag:
           filters:
             tags:
               only: /.*/

.github/workflows/ci-tests.yml

@@ -18,32 +18,29 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
-
-    - name: Start DB
-      run: |
-        docker run --add-host host.docker.internal:host-gateway --name mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=verysecurepw -e MYSQL_DATABASE=test_olympia -d mysql:8.0 --default-authentication-plugin=mysql_native_password
-        sleep 15
-        docker exec mysql mysql -P 3306 -u root -p"verysecurepw" -e "CREATE USER 'root'@'127.0.0.1' IDENTIFIED WITH mysql_native_password BY 'verysecurepw';"
-        docker exec mysql mysql -P 3306 -u root -p"verysecurepw" -e "GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1';"
-
-    - name: Start ES
-      run: docker run --add-host host.docker.internal:host-gateway --name es -p 9200:9200 -p 9300:9300 -e ES_NETWORK_HOST=0.0.0.0 -e discovery.type=single-node -d elasticsearch:5.6.12-alpine
-
-    - name: Run docker container
-      run: docker run --add-host host.docker.internal:host-gateway -it -d --name atn -e DATABASES_DEFAULT_URL=mysql://root:verysecurepw@host.docker.internal:3306/test_olympia -e ES_HOSTS=host.docker.internal:9200 -e ELASTICSEARCH_LOCATION=host.docker.internal:9200 -v $GITHUB_WORKSPACE/src:/data/olympia/src:rw thunderbirdops/addons-server bash
-
-    - name: Install deps
-      run: docker exec atn bash -c "make -f Makefile-docker update_deps"
-
-    - name: Patch Django
-      run:  docker exec atn bash -c "sed -i 's/TX_ISOLATION = /transaction_isolation = /' /usr/local/lib/python2.7/site-packages/django/db/backends/mysql/base.py"
-
-    - name: Run tests
-      run: docker exec atn bash -c 'python -m pytest --junitxml=src/test_report.xml -m "not es_tests" src/olympia/'
-
-    - name: Publish test results
-      uses: EnricoMi/publish-unit-test-result-action@v2
-      if: always()
-      with:
-        files: src/test_report.xml
+      - uses: actions/checkout@v4
+
+      - name: Build Docker Compose
+        run: docker compose build
+
+      - name: Run Docker Compose
+        run: docker compose up -d
+
+      - name: Wait a bit for MySQL to boot
+        run: sleep 10
+
+      - name: Install deps
+        run: docker compose exec web bash -c "make -f Makefile-docker update_deps"
+
+      - name: Patch Django
+        run: docker compose exec web bash -c "sed -i 's/TX_ISOLATION = /transaction_isolation = /' /usr/local/lib/python3.6/site-packages/django/db/backends/mysql/base.py"
+
+      - name: Run tests
+        run: docker compose exec web bash -c 'python -m pytest --junitxml=src/test_report36.xml -m "not es_tests" src/olympia/'
+
+      - name: Publish test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          check_name: "Python 3.6 Test Results"
+          files: src/test_report36.xml

.gitignore

@@ -1,45 +1,49 @@
-.env
-*.egg-info
-settings_local.py*
-settings_local_*.py
-local_settings.py
-shellng_local.py
-*.py[co]
-.*.sw?
-pip-log.txt
-docs/_gh-pages
-docs/api/_build
-build*.py
-.DS_Store
-*.js.tmp
-*.css.tmp
-*-min.css
 *-all.css
-*-min.js
 *-all.js
+*-min.css
+*-min.js
+*.css.tmp
+*.egg-info
+*.js.tmp
 *.less.css
+*.mo
+*.po~
+*.py[co]
+*.signed.zip
 *.styl.css
+.*.sw?
+.cache
+.DS_Store
+.env
+.ipython
 .nose*
-tmp/*
-.pytest_cache/*
-node_modules
 .npm/
-static/css/node_lib/*
-static/js/node_lib/*
-*.signed.zip
-*.po~
-*.mo
-site-static/*
-user-media/*
-guarded-addons/*
+.pytest_cache/*
 .tox/
-MANIFEST
+.vscode
+build*.py
+docker*.yml
 docker/artifacts/*
+docs/_gh-pages
+docs/api/_build
+docs/_build
+local_settings.py
 logs/*
-supervisord.pid
-.ipython
-docker*.yml
-.cache
-.vscode
+MANIFEST
+node_modules
+pip-log.txt
+settings_local.py*
+settings_local_*.py
+shellng_local.py
+site-static/*
 src/olympia/discovery/strings.jinja2
+static/css/node_lib/*
+static/js/i18n/*.js
+static/js/node_lib/*
+storage/files/*
+storage/git-storage/*
+storage/guarded-addons/*
+storage/shared_storage/*
+supervisord.pid
+tmp/*
 venv*

.travis.yml

@@ -1,34 +1,33 @@
 language: python
-sudo: false
+dist: xenial
 
 python:
   - 2.7
 
 jobs:
   fast_finish: true
-  allow_failures:
-    - python: 3.6
   include:
-    - { stage: python2, env: TOXENV=codestyle }
-    - { stage: python2, env: TOXENV=docs }
-    - { stage: python2, env: TOXENV=assets }
-    - { stage: python2, env: TOXENV=es }
-    - { stage: python2, env: TOXENV=addons }
-    - { stage: python2, env: TOXENV=devhub }
-    - { stage: python2, env: TOXENV=reviewers-and-zadmin }
-    - { stage: python2, env: TOXENV=amo-locales-and-signing }
-    - { stage: python2, env: TOXENV=users-accounts-and-ratings }
-    - { stage: python2, env: TOXENV=main }
-    - { stage: python3, python: 3.6, env: TOXENV=codestyle}
-    - { stage: python3, python: 3.6, env: TOXENV=docs }
-    - { stage: python3, python: 3.6, env: TOXENV=assets }
-    - { stage: python3, python: 3.6, env: TOXENV=es }
-    - { stage: python3, python: 3.6, env: TOXENV=addons }
-    - { stage: python3, python: 3.6, env: TOXENV=devhub }
-    - { stage: python3, python: 3.6, env: TOXENV=reviewers-and-zadmin }
-    - { stage: python3, python: 3.6, env: TOXENV=amo-locales-and-signing }
-    - { stage: python3, python: 3.6, env: TOXENV=users-accounts-and-ratings }
-    - { stage: python3, python: 3.6, env: TOXENV=main }
+    - { env: TOXENV=codestyle }
+    - { env: TOXENV=docs }
+    - { env: TOXENV=assets }
+    - { env: TOXENV=es }
+    - { env: TOXENV=addons-versions-and-files }
+    - { env: TOXENV=devhub }
+    - { env: TOXENV=reviewers-and-zadmin }
+    - { env: TOXENV=accounts-users-and-ratings }
+    - { env: TOXENV=amo-lib-locales-and-signing }
+    - { env: TOXENV=main }
+    - { python: 3.6, env: TOXENV=codestyle}
+    - { python: 3.6, env: TOXENV=docs }
+    - { python: 3.6, env: TOXENV=assets }
+    - { python: 3.6, env: TOXENV=es }
+    - { python: 3.6, env: TOXENV=addons-versions-and-files }
+    - { python: 3.6, env: TOXENV=devhub }
+    - { python: 3.6, env: TOXENV=reviewers-and-zadmin }
+    - { python: 3.6, env: TOXENV=accounts-users-and-ratings }
+    - { python: 3.6, env: TOXENV=main }
+    - { python: 3.6, env: TOXENV=amo-lib-locales-and-signing }
+    - { python: 3.6, env: TOXENV=assets  DJANGO=django22 }
 
 env:
   global:
@@ -70,22 +69,23 @@ before_install:
 install:
   - nvm current
   - nvm deactivate
-  - nvm install 10
-  - nvm use 10
-  - pip install --upgrade pip wheel setuptools tox==3.2.1 tox-travis==0.10
+  - nvm install 8
+  - nvm use 8
+  - pip install --no-deps -r requirements/travis_base.txt
   - ./scripts/travis-install-libgit2.sh
 
 before_script:
   - mysql --version
   - node --version
   - java -version
   - curl -v http://localhost:9200/
+  - sudo touch /addons-server-docker-container
 
 script:
   - |
     if [[ $TRAVIS_EVENT_TYPE != "cron" ]]; then
-      if [[ $TOXENV == "amo-locales-and-signing" ]]; then
-         docker run --name autograph -d -p 5500:5500 -v $(pwd)/scripts/:/scripts/ mozilla/autograph:2.3.2 /go/bin/autograph -c /scripts/autograph_travis_test_config.yaml
+      if [[ $TOXENV == "amo-lib-locales-and-signing" ]]; then
+         docker run --name autograph -d -p 5500:5500 -v $(pwd)/scripts/:/scripts/ mozilla/autograph:2.7.0 /go/bin/autograph -c /scripts/autograph_travis_test_config.yaml
       fi
       RUNNING_IN_CI=True tox
     fi