[staged-updates] Replaced UWSGI with Gunicorn.

After an unreasonable amount of effort with a bug in uwsgi that would cause random 500, 502, 503, and 504 errors
I have decided to replace it with Gunicorn, despite my own preference for uwsgi.  This situation was deeply
debugged and no combination of working settings was found.  There are open bugs in UWSGI about similar errors going
back for years, and while some people have found solutions virtually nothing I changed had an effect.
THE ONLY effect I could force was the statistical prevelance of which 500 code it tended towards by setting
up Apache and uwsgi to use the uwsgi protocol instead of http, which dropped 502s in favor of 503s.
After switching to (and this time correctly configuring without a critical misread of some documentation)
Gunicorn, this time it works and does not unceremoniously kill processes.  500 class errors dropped to zero
from hundreds per hour on our production server.

.ebextensions/01.config

@@ -23,7 +23,7 @@ commands:
 container_commands:
   01_setup_profile:
     # quality of life for ssh sessions
-    command: mv ./cluster_management/pushed_files/eb_profile.sh /home/ec2-user/.bashrc; chmod 644 /home/ec2-user/.bashrc; chown ec2-user /home/ec2-user/.bashrc; chgrp ec2-user /home/ec2-user/.bashrc
+    command: mv ./cluster_management/pushed_files/eb_profile.sh /home/ec2-user/.bash_profile; chmod 644 /home/ec2-user/.bash_profile; chown ec2-user /home/ec2-user/.bash_profile; chgrp ec2-user /home/ec2-user/.bash_profile
   02_setup_reasonable_inputrc:
     command: mv ./cluster_management/pushed_files/.inputrc /home/ec2-user/.inputrc; chmod 664 /home/ec2-user/.inputrc; chown ec2-user /home/ec2-user/.inputrc; chgrp ec2-user /home/ec2-user/.inputrc
   03_migrate:
@@ -49,4 +49,4 @@ option_settings:
 
 # the location to stick custom apache configurations is at /etc/httpd/conf.d/elasticbeanstalk/*.conf
 #  (as far as I can tell, anyway.  its declared in the default apache configuration.  )
-#to change the number of processes and threads check the ProcFile parameters
+#to change the number of processes and threads check the ProcFile parameters

Procfile

@@ -1 +1 @@
-web: uwsgi --yaml uwsgi-config.yml
+web: gunicorn --config=gunicorn_conf.py

cluster_management/pushed_files/bash_profile.sh

@@ -36,6 +36,7 @@ alias deleteswap="sudo swapoff /swapfile; sudo rm /swapfile"
 
 #Bash Utility
 alias sudo="sudo " # allows the use of all our aliases with the sudo command
+alias watch="watch "
 alias n='nano -c' # nano with syntax highlighting
 alias no="nano -Iwn" # nano in overwrite mode to allow for quick pasted overwrites
 alias sn='sudo nano -c'
@@ -76,8 +77,8 @@ alias processing-stop="killall supervisord > /dev/null 2>&1"
 alias processing-restart="pkill -HUP supervisord 2> /dev/null"
 
 #Watch Logs Live
-alias log='tail -f /home/ubuntu/celery*.log'
-alias logs='tail -f /home/ubuntu/celery*.log'
+alias log='tail -f /home/ubuntu/celery*.log /home/ubuntu/supervisor.log'
+alias logs='logs'
 alias logd='tail -f /home/ubuntu/supervisor.log'
 
 #Configuration files

cluster_management/pushed_files/eb_profile.sh

@@ -12,47 +12,92 @@ source /var/app/venv/*/bin/activate
 
 `/opt/elasticbeanstalk/bin/get-config optionsettings | jq '."aws:elasticbeanstalk:application:environment"' | jq -r 'to_entries | .[] | "export \(.key)=\(.value)"'`
 
+# cd into the current app directory
 cd /var/app
 cd /var/app/current
 
+
+# location aliases
+alias b='cd /var/app; cd /var/app/current'
+alias conf='cd /etc/httpd/conf.d/elasticbeanstalk'
+
+
 export EDITOR='nano'
 
-alias db='cd /var/app/current; python /var/app/current/manage.py shell_plus'
-alias restart='sudo killall -s 1 supervisord; htop -u apache'
 
-alias log='sudo tail -f /var/log/httpd/access_log /var/log/httpd/error_log /var/log/cfn-* /var/log/eb-* /var/log/messages'
+
+echo '
+Important log files and aliases:
+loga:  /var/log/httpd/access_log /var/log/httpd/error_log   # httpd
+logm:  /var/log/messages                                    # system messages
+logu:  /var/log/web.stdout.log                              # gunicorn/django/gunicorn
+logeb: /var/log/cfn-... /var/log/eb-...                     # AWS
+logw - follows not the eb logs
+log, logs - watch all logs
+'
+
+## platform log file notes
+# eb-engine.log     # the useful one
+# eb-cfn-init.log   # some output about deploys 1
+# eb-publish.log    # some output about deploys 2
+
+# cfn-hup.log       # "command processing is alive" (junk)
+# cfn-init-cmd.log  # real output from deployment operations
+# cfn-init.log      # partial output from deployment operations?  
+# cfn-wire.log      # junk, sqs stuff
+
+
+#Watch Logs Live
 alias logs='log'
-alias log-important='sudo tail -f /var/log/httpd/access_log /var/log/httpd/error_log /var/log/messages'
-alias logi='log-important'
-alias logimportant='log-important'
-alias log_important='log-important'
-alias logeb="/var/log/cfn-* /var/log/eb-*"
-alias loghttpd='tail -f /var/log/httpd/*'
-alias logdjango='sudo tail -f  /var/log/web.stdout.log'
-alias logd='logdjango'
-alias loggunicorn='sudo tail -f /var/log/messages'
-alias logg='loggunicorn'
+alias log='sudo tail -f /var/log/httpd/access_log /var/log/httpd/error_log /var/log/messages /var/log/web.stdout.log /var/log/cfn-init-cmd.log /var/log/cfn-init.log /var/log/eb-engine.log /var/log/eb-cfn-init.log /var/log/eb-publish.log'
+alias logw='sudo tail -f /var/log/httpd/access_log /var/log/httpd/error_log /var/log/web.stdout.log'  # mostly the web stuff
 
-alias sudo="sudo "
-alias n="nano "
-alias no="nano -Iwn "
-alias sn="sudo nano "
-alias sno="sudo nano -Iwn "
+alias logaws="logeb"
+alias logdeploy="logeb"
+alias logeb="sudo tail -f /var/log/cfn-init-cmd.log /var/log/cfn-init.log /var/log/eb-engine.log /var/log/eb-cfn-init.log /var/log/eb-publish.log"
 
-alias pyc='find . -type f -name "*.pyc" -delete -print'
-alias htop="htop -d 5"
+alias logh='loghttpd'
+alias loga='loghttpd'
+alias logapache='loghttpd'
+alias loghttpd='sudo tail -f /var/log/httpd/*'
 
-alias u="cd .."
-alias uu="cd ../.."
-alias uuu="cd ../../.."
+alias logd='logdjango'
+alias logu='logdjango'
+alias logdjango='sudo tail -f  /var/log/web.stdout.log'  # I guess it's sudo to protect it?
 
-alias ls='ls --color=auto'
+alias logm='loggunicorn'
+alias logmessages='sudo tail -f /var/log/messages'
+
+# Bash Utility
+alias sudo="sudo " # allows the use of all our aliases with the sudo command
+alias watch="watch "
+alias n='nano -c' # nano with syntax highlighting
+alias no="nano -Iwn" # nano in overwrite mode to allow for quick pasted overwrites
+alias sn='sudo nano -c'
+alias sno="sudo nano -Iwn"
+alias ls='ls --color=auto -h'
 alias l='ls'
 alias la='ls -A'
-alias ll='ls -Alh'
+alias lla='ll -A'
+alias ll='ls -lh'
+alias lg='ls -Alh | grep -i '
 alias lh='ls -lhX --color=auto'
+alias lll="du -ah --max-depth=0 --block-size=MB --time * | sort -nr"
+alias slll="sudo du -ah --max-depth=0 --block-size=MB --time * | sort -nr"
+alias grep='grep --color=auto' # make grep not suck
+alias g='grep -i' # single letter for case insensitive grep
+alias u="cd .." # navigate directories.
+alias uu="cd ../.."
+alias uuu="cd ../../.."
 
-alias py="python"
+#Developer tools
+alias db='cd /var/app/current; python /var/app/current/manage.py shell_plus'
 alias ipy="ipython"
+alias manage="python manage.py"
+alias shell="python manage.py shell_plus"
+alias showmigrations='manage showmigrations'
+alias ag="clear; printf '_%.0s' {1..100}; echo ''; echo 'Silver results begin here:'; ag --column"
+alias pyc='find . -type f -name "*.pyc" -delete -print'
 
-alias ls='ls --color=auto -h'
+#htop!
+alias htop="htop -d 5"

endpoints/mobile_endpoints.py

@@ -347,7 +347,7 @@ def contains_valid_extension(file_name: str):
 
 
 @determine_os_api
-@authenticate_participant
+@minimal_validation
 def get_latest_device_settings(request: ParticipantRequest, OS_API=""):
     """ Extremely simple endpoint that returns the device settings for the study as a json string. 
     Endpoint is used by the app to periodically check for changes to the device settings. """
@@ -360,7 +360,7 @@ def get_latest_device_settings(request: ParticipantRequest, OS_API=""):
 
 
 @determine_os_api
-@authenticate_participant
+@minimal_validation
 def get_latest_surveys(request: ParticipantRequest, OS_API=""):
     """ This is the endpoint hit by the app to download the current survey and survey schedule 
     information.  The app's representation of surveys is of the current week. """
@@ -446,7 +446,7 @@ def format_survey_for_device(survey: Survey, participant: Participant):
 #   of the validation error, which actually occurs at the get-or-create line resulting in the bug.
 #  Probably use a transaction?
 @require_POST
-@authenticate_participant
+@minimal_validation
 def set_fcm_token(request: ParticipantRequest):
     """ Sets a participants Firebase Cloud Messaging (FCM) instance token, called whenever a new
     token is generated. Expects a patient_id and and fcm_token in the request body. """

requirements.in

@@ -1,9 +1,9 @@
 # core libraries
-Django>=4.2.17
+Django>=4.2.19
 Django<5.0.0
 firebase-admin==5.4.0
 Jinja2 # used to require pinning, I guess review if there is a major version bump.
-zstd>=1.5.5.1 # can be stuck on older versions sometimes
+zstd>=1.5.6.1 # can be stuck on older versions sometimes - 1.5.6.2 got yanked due to build problem
 orjson==3.9.15
 
 # django-extensions for some really nice management commands and terminal helpers.
@@ -23,7 +23,7 @@ pypng
 pyotp
 
 # This version of pycryptodome results in "from Cryptodome import ..." imports.
-pycryptodomex==3.20.0  # locked version, only update this manuallly, has had breaking changes in the past
+pycryptodomex==3.21.0  # locked version, only update this manuallly, has had breaking changes in the past
 
 # error reporting
 cronutils==0.4.2
@@ -37,7 +37,7 @@ psycopg>=3.1.18
 # ipython can exhibit weird issues that make it look like the codebase is broken.  Most issues
 # are resolved by forcing this relatively new version of ipython.
 ipython>=8.12.2
-uwsgi>=2.0.24
+gunicorn
 
 # Keep these dependencies up to date
 boto3

requirements.txt

@@ -16,9 +16,9 @@ billiard==4.2.1
     # via celery
 bleach==6.2.0
     # via -r requirements.in
-boto3==1.36.8
+boto3==1.36.20
     # via -r requirements.in
-botocore==1.36.8
+botocore==1.36.20
     # via
     #   boto3
     #   s3transfer
@@ -28,7 +28,7 @@ cachetools==5.5.1
     # via google-auth
 celery==5.4.0
     # via -r requirements.in
-certifi==2024.12.14
+certifi==2025.1.31
     # via
     #   pyproj
     #   requests
@@ -57,7 +57,7 @@ decorator==5.1.1
     # via
     #   ipython
     #   librosa
-django==4.2.18
+django==4.2.19
     # via
     #   -r requirements.in
     #   django-extensions
@@ -79,7 +79,7 @@ google-api-core[grpc]==2.24.1
     #   google-cloud-core
     #   google-cloud-firestore
     #   google-cloud-storage
-google-api-python-client==2.160.0
+google-api-python-client==2.161.0
     # via firebase-admin
 google-auth==2.38.0
     # via
@@ -97,15 +97,15 @@ google-cloud-core==2.4.1
     #   google-cloud-storage
 google-cloud-firestore==2.20.0
     # via firebase-admin
-google-cloud-storage==2.19.0
+google-cloud-storage==3.0.0
     # via firebase-admin
 google-crc32c==1.6.0
     # via
     #   google-cloud-storage
     #   google-resumable-media
 google-resumable-media==2.7.2
     # via google-cloud-storage
-googleapis-common-protos==1.66.0
+googleapis-common-protos==1.67.0
     # via
     #   google-api-core
     #   grpcio-status
@@ -115,17 +115,19 @@ grpcio==1.70.0
     #   grpcio-status
 grpcio-status==1.70.0
     # via google-api-core
-h3==4.2.0
+gunicorn==23.0.0
+    # via -r requirements.in
+h3==4.2.1
     # via timezonefinder
-holidays==0.65
+holidays==0.66
     # via forest
 httplib2==0.22.0
     # via
     #   google-api-python-client
     #   google-auth-httplib2
 idna==3.10
     # via requests
-ipython==8.31.0
+ipython==8.32.0
     # via -r requirements.in
 jedi==0.19.2
     # via ipython
@@ -178,6 +180,7 @@ orjson==3.9.15
     # via -r requirements.in
 packaging==24.2
     # via
+    #   gunicorn
     #   lazy-loader
     #   pooch
 pandas==2.2.3
@@ -219,7 +222,7 @@ pyasn1-modules==0.4.1
     # via google-auth
 pycparser==2.22
     # via cffi
-pycryptodomex==3.20.0
+pycryptodomex==3.21.0
     # via -r requirements.in
 pygments==2.19.1
     # via ipython
@@ -240,7 +243,7 @@ python-dateutil==2.9.0.post0
     #   celery
     #   holidays
     #   pandas
-pytz==2024.2
+pytz==2025.1
     # via
     #   django-timezone-field
     #   forest
@@ -267,9 +270,9 @@ scipy==1.15.1
     #   librosa
     #   scikit-learn
     #   ssqueezepy
-sentry-sdk==2.20.0
+sentry-sdk==2.21.0
     # via cronutils
-shapely==2.0.6
+shapely==2.0.7
     # via forest
 six==1.17.0
     # via python-dateutil
@@ -307,8 +310,6 @@ urllib3==2.3.0
     #   botocore
     #   requests
     #   sentry-sdk
-uwsgi==2.0.28
-    # via -r requirements.in
 vine==5.1.0
     # via
     #   amqp
@@ -320,5 +321,5 @@ webencodings==0.5.1
     # via bleach
 wheel==0.45.1
     # via forest
-zstd==1.5.6.2
+zstd==1.5.6.1
     # via -r requirements.in