Skip to content

Commit

Permalink
Merge upstream HEAD (da6668b, 2024-01-02) for auth fix
Browse files Browse the repository at this point in the history 
The recently cherry-picked CVE-2023-51663 fix included refactored
code dependent on recent upstream code that was not cherry-picked.
Merge current upstream HEAD to include the necessary AppKeys class
and any unrelated fixes and improvements.
  • Loading branch information
@jmarshall
jmarshall committed Jan 3, 2024
2 parents e5b3147 + da6668b commit f3d6d7f
Show file tree
Hide file tree
Showing 171 changed files with 4,803 additions and 2,724 deletions.
8 changes: 6 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ SERVICES_IMAGES := $(patsubst %, %-image, $(SERVICES_PLUS_ADMIN_POD))
SERVICES_DATABASES := $(patsubst %, %-db, $(SERVICES))
SERVICES_MODULES := $(SERVICES) gear web_common
CHECK_SERVICES_MODULES := $(patsubst %, check-%, $(SERVICES_MODULES))
SPECIAL_IMAGES := hail-ubuntu batch-worker
SPECIAL_IMAGES := hail-ubuntu batch-worker letsencrypt

HAILGENETICS_IMAGES = $(foreach img,hail vep-grch37-85 vep-grch38-95,hailgenetics-$(img))
CI_IMAGES = ci-utils ci-buildkit base hail-run
Expand Down Expand Up @@ -118,7 +118,7 @@ hail/python/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt
hail/python/dev/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/requirements.txt
./generate-linux-pip-lockfile.sh hail/python/dev

benchmark/python/pinned-requirements.txt: benchmark/python/requirements.txt
benchmark/python/pinned-requirements.txt: benchmark/python/requirements.txt hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt
./generate-linux-pip-lockfile.sh benchmark/python

gear/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
Expand Down Expand Up @@ -217,6 +217,10 @@ hailgenetics-vep-grch38-95-image: hail-ubuntu-image
--build-arg BASE_IMAGE=$(shell cat hail-ubuntu-image)
echo $(IMAGE_NAME) > $@

letsencrypt-image:
./docker-build.sh letsencrypt Dockerfile $(IMAGE_NAME)
echo $(IMAGE_NAME) > $@

$(PRIVATE_REGISTRY_IMAGES): pushed-private-%-image: %-image
! [ -z $(NAMESPACE) ] # call this like: make ... NAMESPACE=default
[ $(DOCKER_PREFIX) != docker.io ] # DOCKER_PREFIX should be an internal private registry
Expand Down
89 changes: 49 additions & 40 deletions auth/auth/auth.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ async def get_index(request: web.Request, userdata: Optional[UserData]) -> web.R
@routes.get('/creating')
@auth.maybe_authenticated_user
async def creating_account(request: web.Request, userdata: Optional[UserData]) -> web.Response:
db = request.app['db']
db = request.app[AppKeys.DB]
session = await aiohttp_session.get_session(request)
if 'pending' in session:
login_id = session['login_id']
Expand Down Expand Up @@ -256,7 +256,7 @@ async def creating_account_wait(request):

async def _wait_websocket(request, login_id):
app = request.app
db = app['db']
db = app[AppKeys.DB]

user = await user_from_login_id(db, login_id)
if not user:
Expand Down Expand Up @@ -290,7 +290,7 @@ async def _wait_websocket(request, login_id):
async def signup(request) -> NoReturn:
next_page = request.query.get('next', deploy_config.external_url('auth', '/user'))

flow_data = request.app['flow_client'].initiate_flow(deploy_config.external_url('auth', '/oauth2callback'))
flow_data = request.app[AppKeys.FLOW_CLIENT].initiate_flow(deploy_config.external_url('auth', '/oauth2callback'))

session = await aiohttp_session.new_session(request)
cleanup_session(session)
Expand All @@ -305,7 +305,7 @@ async def signup(request) -> NoReturn:
async def login(request) -> NoReturn:
next_page = request.query.get('next', deploy_config.external_url('auth', '/user'))

flow_data = request.app['flow_client'].initiate_flow(deploy_config.external_url('auth', '/oauth2callback'))
flow_data = request.app[AppKeys.FLOW_CLIENT].initiate_flow(deploy_config.external_url('auth', '/oauth2callback'))

session = await aiohttp_session.new_session(request)
cleanup_session(session)
Expand Down Expand Up @@ -341,7 +341,7 @@ async def callback(request) -> web.Response:
log.exception('oauth2 callback: could not fetch and verify token')
raise web.HTTPUnauthorized() from e

db = request.app['db']
db = request.app[AppKeys.DB]

user = await user_from_login_id(db, login_id)

Expand Down Expand Up @@ -394,7 +394,7 @@ async def callback(request) -> web.Response:
@routes.post('/api/v1alpha/users/{user}/create')
@auth.authenticated_developers_only()
async def create_user(request: web.Request, _) -> web.Response:
db: Database = request.app['db']
db = request.app[AppKeys.DB]
username = request.match_info['user']

body = await json_request(request)
Expand Down Expand Up @@ -443,7 +443,7 @@ async def create_copy_paste_token(db, session_id, max_age_secs=300):
async def get_copy_paste_token(request: web.Request, userdata: UserData) -> web.Response:
session = await aiohttp_session.get_session(request)
session_id = session['session_id']
db = request.app['db']
db = request.app[AppKeys.DB]
copy_paste_token = await create_copy_paste_token(db, session_id)
page_context = {'copy_paste_token': copy_paste_token}
return await render_template('auth', request, userdata, 'copy-paste-token.html', page_context)
Expand All @@ -453,7 +453,7 @@ async def get_copy_paste_token(request: web.Request, userdata: UserData) -> web.
@auth.authenticated_users_only()
async def get_copy_paste_token_api(request: web.Request, _) -> web.Response:
session_id = await get_session_id(request)
db = request.app['db']
db = request.app[AppKeys.DB]
copy_paste_token = await create_copy_paste_token(db, session_id)
return web.Response(body=copy_paste_token)

Expand All @@ -464,7 +464,7 @@ async def logout(request: web.Request, userdata: Optional[UserData]) -> NoReturn
if not userdata:
raise web.HTTPFound(deploy_config.external_url('auth', ''))

db = request.app['db']
db = request.app[AppKeys.DB]
session_id = await get_session_id(request)
await db.just_execute('DELETE FROM sessions WHERE session_id = %s;', session_id)

Expand All @@ -478,7 +478,7 @@ async def logout(request: web.Request, userdata: Optional[UserData]) -> NoReturn
async def rest_login(request: web.Request) -> web.Response:
callback_port = request.query['callback_port']
callback_uri = f'http://127.0.0.1:{callback_port}/oauth2callback'
flow_data = request.app['flow_client'].initiate_flow(callback_uri)
flow_data = request.app[AppKeys.FLOW_CLIENT].initiate_flow(callback_uri)
flow_data['callback_uri'] = callback_uri

# keeping authorization_url and state for backwards compatibility
Expand All @@ -490,13 +490,13 @@ async def rest_login(request: web.Request) -> web.Response:
@routes.get('/api/v1alpha/oauth2-client')
async def hailctl_oauth_client(request): # pylint: disable=unused-argument
idp = IdentityProvider.GOOGLE if CLOUD == 'gcp' else IdentityProvider.MICROSOFT
return json_response({'idp': idp.value, 'oauth2_client': request.app['hailctl_client_config']})
return json_response({'idp': idp.value, 'oauth2_client': request.app[AppKeys.HAILCTL_CLIENT_CONFIG]})


@routes.get('/roles')
@auth.authenticated_developers_only()
async def get_roles(request: web.Request, userdata: UserData) -> web.Response:
db = request.app['db']
db = request.app[AppKeys.DB]
roles = [x async for x in db.select_and_fetchall('SELECT * FROM roles;')]
page_context = {'roles': roles}
return await render_template('auth', request, userdata, 'roles.html', page_context)
Expand All @@ -506,7 +506,7 @@ async def get_roles(request: web.Request, userdata: UserData) -> web.Response:
@auth.authenticated_developers_only()
async def post_create_role(request: web.Request, _) -> NoReturn:
session = await aiohttp_session.get_session(request)
db = request.app['db']
db = request.app[AppKeys.DB]
post = await request.post()
name = str(post['name'])

Expand All @@ -526,7 +526,7 @@ async def post_create_role(request: web.Request, _) -> NoReturn:
@routes.get('/users')
@auth.authenticated_developers_only()
async def get_users(request: web.Request, userdata: UserData) -> web.Response:
db = request.app['db']
db = request.app[AppKeys.DB]
users = [x async for x in db.select_and_fetchall('SELECT * FROM users;')]
page_context = {'users': users}
return await render_template('auth', request, userdata, 'users.html', page_context)
Expand All @@ -536,7 +536,7 @@ async def get_users(request: web.Request, userdata: UserData) -> web.Response:
@auth.authenticated_developers_only()
async def post_create_user(request: web.Request, _) -> NoReturn:
session = await aiohttp_session.get_session(request)
db = request.app['db']
db = request.app[AppKeys.DB]
post = await request.post()
username = str(post['username'])
login_id = str(post['login_id']) if 'login_id' in post else None
Expand All @@ -563,7 +563,7 @@ async def rest_get_users(request: web.Request, userdata: UserData) -> web.Respon
if userdata['is_developer'] != 1 and userdata['username'] != 'ci':
raise web.HTTPUnauthorized()

db: Database = request.app['db']
db = request.app[AppKeys.DB]
_query = '''
SELECT id, username, login_id, state, is_developer, is_service_account, hail_identity
FROM users;
Expand All @@ -575,7 +575,7 @@ async def rest_get_users(request: web.Request, userdata: UserData) -> web.Respon
@routes.get('/api/v1alpha/users/{user}')
@auth.authenticated_developers_only()
async def rest_get_user(request: web.Request, _) -> web.Response:
db: Database = request.app['db']
db = request.app[AppKeys.DB]
username = request.match_info['user']

user = await db.select_and_fetchone(
Expand Down Expand Up @@ -615,7 +615,7 @@ async def _delete_user(db: Database, username: str, id: Optional[str]):
@auth.authenticated_developers_only()
async def delete_user(request: web.Request, _) -> NoReturn:
session = await aiohttp_session.get_session(request)
db = request.app['db']
db = request.app[AppKeys.DB]
post = await request.post()
id = str(post['id'])
username = str(post['username'])
Expand All @@ -632,7 +632,7 @@ async def delete_user(request: web.Request, _) -> NoReturn:
@routes.delete('/api/v1alpha/users/{user}')
@auth.authenticated_developers_only()
async def rest_delete_user(request: web.Request, _) -> web.Response:
db = request.app['db']
db = request.app[AppKeys.DB]
username = request.match_info['user']

try:
Expand All @@ -657,14 +657,14 @@ async def rest_callback(request):
flow_dict = json.loads(request.query['flow'])

try:
flow_result = request.app['flow_client'].receive_callback(request, flow_dict)
flow_result = request.app[AppKeys.FLOW_CLIENT].receive_callback(request, flow_dict)
except asyncio.CancelledError:
raise
except Exception as e:
log.exception('fetching and decoding token')
raise web.HTTPUnauthorized() from e

db = request.app['db']
db = request.app[AppKeys.DB]
users = [
x
async for x in db.select_and_fetchall(
Expand All @@ -684,7 +684,7 @@ async def rest_callback(request):
@routes.post('/api/v1alpha/copy-paste-login')
async def rest_copy_paste_login(request):
copy_paste_token = request.query['copy_paste_token']
db = request.app['db']
db = request.app[AppKeys.DB]

@transaction(db)
async def maybe_pop_token(tx):
Expand All @@ -711,21 +711,21 @@ async def maybe_pop_token(tx):
@auth.authenticated_users_only()
async def rest_logout(request: web.Request, _) -> web.Response:
session_id = await get_session_id(request)
db = request.app['db']
db = request.app[AppKeys.DB]
await db.just_execute('DELETE FROM sessions WHERE session_id = %s;', session_id)

return web.Response(status=200)


async def get_userinfo(request: web.Request, auth_token: str) -> UserData:
flow_client: Flow = request.app['flow_client']
client_session = request.app['client_session']
flow_client = request.app[AppKeys.FLOW_CLIENT]
client_session = request.app[AppKeys.CLIENT_SESSION]

userdata = await get_userinfo_from_hail_session_id(request, auth_token)
if userdata:
return userdata

hailctl_oauth_client = request.app['hailctl_client_config']
hailctl_oauth_client = request.app[AppKeys.HAILCTL_CLIENT_CONFIG]
uid = await flow_client.get_identity_uid_from_access_token(
client_session, auth_token, oauth2_client=hailctl_oauth_client
)
Expand All @@ -738,7 +738,7 @@ async def get_userinfo(request: web.Request, auth_token: str) -> UserData:
async def get_userinfo_from_login_id_or_hail_identity_id(
request: web.Request, login_id_or_hail_idenity_uid: str
) -> UserData:
db = request.app['db']
db = request.app[AppKeys.DB]

users = [
x
Expand All @@ -755,15 +755,15 @@ async def get_userinfo_from_login_id_or_hail_identity_id(
if len(users) != 1:
log.info('Unknown login id')
raise web.HTTPUnauthorized()
return users[0]
return typing.cast(UserData, users[0])


async def get_userinfo_from_hail_session_id(request: web.Request, session_id: str) -> Optional[UserData]:
# b64 encoding of 32-byte session ID is 44 bytes
if len(session_id) != 44:
return None

db = request.app['db']
db = request.app[AppKeys.DB]
users = [
x
async for x in db.select_and_fetchall(
Expand All @@ -780,7 +780,7 @@ async def get_userinfo_from_hail_session_id(request: web.Request, session_id: st

if len(users) != 1:
return None
return users[0]
return typing.cast(UserData, users[0])


@routes.get('/api/v1alpha/userinfo')
Expand All @@ -805,32 +805,41 @@ async def verify_dev_or_sa_credentials(_, userdata: UserData) -> web.Response:
return web.Response(status=200)


class AppKeys:
DB = web.AppKey('db', Database)
CLIENT_SESSION = web.AppKey('client_session', httpx.ClientSession)
FLOW_CLIENT = web.AppKey('flow_client', Flow)
HAILCTL_CLIENT_CONFIG = web.AppKey('hailctl_client_config', dict)
K8S_CLIENT = web.AppKey('k8s_client', kubernetes_asyncio.client.CoreV1Api)
K8S_CACHE = web.AppKey('k8s_cache', K8sCache)


async def on_startup(app):
db = Database()
await db.async_init(maxsize=50)
app['db'] = db
app['client_session'] = httpx.client_session()
app[AppKeys.DB] = db
app[AppKeys.CLIENT_SESSION] = httpx.client_session()

credentials_file = '/auth-oauth2-client-secret/client_secret.json'
if CLOUD == 'gcp':
app['flow_client'] = GoogleFlow(credentials_file)
app[AppKeys.FLOW_CLIENT] = GoogleFlow(credentials_file)
else:
assert CLOUD == 'azure'
app['flow_client'] = AzureFlow(credentials_file)
app[AppKeys.FLOW_CLIENT] = AzureFlow(credentials_file)

with open('/auth-oauth2-client-secret/hailctl_client_secret.json', 'r', encoding='utf-8') as f:
app['hailctl_client_config'] = json.loads(f.read())
app[AppKeys.HAILCTL_CLIENT_CONFIG] = json.loads(f.read())

kubernetes_asyncio.config.load_incluster_config()
app['k8s_client'] = kubernetes_asyncio.client.CoreV1Api()
app['k8s_cache'] = K8sCache(app['k8s_client'])
app[AppKeys.K8S_CLIENT] = kubernetes_asyncio.client.CoreV1Api()
app[AppKeys.K8S_CACHE] = K8sCache(app[AppKeys.K8S_CLIENT])


async def on_cleanup(app):
async with AsyncExitStack() as cleanup:
cleanup.push_async_callback(app['k8s_client'].api_client.rest_client.pool_manager.close)
cleanup.push_async_callback(app['db'].async_close)
cleanup.push_async_callback(app['client_session'].close)
cleanup.push_async_callback(app[AppKeys.K8S_CLIENT].api_client.rest_client.pool_manager.close)
cleanup.push_async_callback(app[AppKeys.DB].async_close)
cleanup.push_async_callback(app[AppKeys.CLIENT_SESSION].close)


class AuthAccessLogger(AccessLogger):
Expand Down
13 changes: 4 additions & 9 deletions batch/batch/cloud/gcp/driver/create_instance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,25 +240,20 @@ def scheduling() -> dict:
labels.instance_id:
static_value: $INSTANCE_ID
service:
log_level: error
pipelines:
default_pipeline:
processors: [labels]
receivers: [runlog, workerlog, jvmlog]
metrics:
receivers:
hostmetrics:
type: hostmetrics
collection_interval: 60s
processors:
metrics_filter:
type: exclude_metrics
metrics_pattern: []
metrics_pattern:
- agent.googleapis.com/*/*
service:
pipelines:
default_pipeline:
receivers: [hostmetrics]
processors: [metrics_filter]
log_level: error
EOF
sudo systemctl restart google-cloud-ops-agent
Expand Down
Loading

0 comments on commit f3d6d7f

Please sign in to comment.