Merge pull request #324 from chreekat/b/handover-patches

Handover patches

LICENSE

@@ -1,6 +1,7 @@
 The MIT License (MIT)
 
 Copyright (c) 2014-2017 FP Complete
+Copyright (c) 2024 Haskell Foundation
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

app/stackage-server-cron.hs

@@ -38,17 +38,24 @@ optsParser =
              \their yaml files from stackage-snapshots repo have been updated or not.") <*>
     option
         readText
-        (long "download-bucket" <> value haddockBucketName <> metavar "DOWNLOAD_BUCKET" <>
+        (long "download-bucket" <> value defHaddockBucketName <> metavar "DOWNLOAD_BUCKET" <>
          help
              ("S3 Bucket name where things like haddock and current hoogle files should \
-              \be downloaded from. Default is: " <>
-              T.unpack haddockBucketName)) <*>
+              \be downloaded from. Used in S3 API read operations. Default is: " <>
+              T.unpack defHaddockBucketName)) <*>
     option
         readText
-        (long "upload-bucket" <> value haddockBucketName <> metavar "UPLOAD_BUCKET" <>
+        (long "download-bucket-url" <> value defHaddockBucketUrl <> metavar "DOWNLOAD_BUCKET_URL" <>
+         help
+             ("Publicly accessible URL where the download bucket can be accessed. Used for \
+              \serving the Haddocks on the website. Default is: " <>
+              T.unpack defHaddockBucketUrl)) <*>
+    option
+        readText
+        (long "upload-bucket" <> value defHaddockBucketName <> metavar "UPLOAD_BUCKET" <>
          help
              ("S3 Bucket where hoogle db and snapshots.json file will be uploaded to. Default is: " <>
-              T.unpack haddockBucketName)) <*>
+              T.unpack defHaddockBucketName)) <*>
     switch
         (long "do-not-upload" <>
          help "Stop from hoogle db and snapshots.json from being generated and uploaded") <*>

config/settings.yml

@@ -24,3 +24,6 @@ force-ssl: false
 
 postgres-string: "_env:PGSTRING:host=localhost port=5432 user=stackage dbname=stackage password=stackage"
 postgres-poolsize: "_env:PGPOOLSIZE:8"
+
+# Publicly-accessible URL for the bucket holding Haddock contents.
+download-bucket-url: "_env:DOWNLOAD_BUCKET_URL:https://s3.amazonaws.com/haddock.stackage.org"

src/Application.hs

@@ -157,7 +157,7 @@ withFoundation appLogFunc appSettings inner = do
                   runRIO appLogFunc $ RIO.logError $ "Couldn't get Stack matcher: " <> displayShow e
                   pure oldMatcher
         appMirrorStatus <- mkUpdateMirrorStatus
-        hoogleLocker <- newHoogleLocker appLogFunc appHttpManager
+        hoogleLocker <- newHoogleLocker appLogFunc appHttpManager (appDownloadBucketUrl appSettings)
         let appGetHoogleDB = singleRun hoogleLocker
         let appGitRev = $$tGitRev
         runConcurrently $ runContentUpdates *> Concurrently (inner App {..})

src/Control/SingleRun.hs

@@ -84,6 +84,12 @@ singleRun sr@(SingleRun var f) k =
 
                         -- OK, we're done running, so let other
                         -- threads run this again.
+
+                        -- NB: as soon as we've modified the MVar, the next
+                        -- call to singleRun will think no thread is working and
+                        -- start over. Anything waiting on us will get our
+                        -- result, but nobody else will. That's ok: singleRun
+                        -- just provides a little caching on top of a mutex.
                         modifyMVar_ var $ return . filter (\(k', _) -> k /= k')
 
                         case eres of

src/Data/WebsiteContent.hs

@@ -2,6 +2,7 @@
 {-# LANGUAGE NoImplicitPrelude #-}
 {-# LANGUAGE RecordWildCards #-}
 {-# LANGUAGE TypeFamilies #-}
+{-# LANGUAGE TypeOperators #-}
 module Data.WebsiteContent
     ( WebsiteContent (..)
     , StackRelease (..)

src/Handler/DownloadStack.hs

@@ -11,6 +11,7 @@ import Data.Conduit.Attoparsec (sinkParser)
 import Data.WebsiteContent
 import Import
 import Yesod.GitRepo
+import qualified Data.Aeson.KeyMap as Aeson
 
 getDownloadStackListR :: Handler Html
 getDownloadStackListR = track "Handler.DownloadStack.getDownloadStackListR" $ do
@@ -35,14 +36,14 @@ getLatestMatcher man = do
     return $ \pattern' -> do
         let pattern'' = pattern' ++ "."
         Object top <- return val
-        Array assets <- lookup "assets" top
+        Array assets <- Aeson.lookup "assets" top
         headMay $ preferZip $ catMaybes $ map (findMatch pattern'') assets
   where
     findMatch pattern' (Object o) = do
-        String name <- lookup "name" o
+        String name <- Aeson.lookup "name" o
         guard $ not $ ".asc" `isSuffixOf` name
         guard $ pattern' `isInfixOf` name
-        String url <- lookup "browser_download_url" o
+        String url <- Aeson.lookup "browser_download_url" o
         Just url
     findMatch _ _ = Nothing
 

src/Handler/Haddock.hs

@@ -8,13 +8,14 @@ import Import
 import qualified Data.Text as T (takeEnd)
 import Stackage.Database
 
-makeURL :: SnapName -> [Text] -> Text
-makeURL snapName rest = concat
-    $ "https://s3.amazonaws.com/"
-    : haddockBucketName
-    : "/"
-    : toPathPiece snapName
-    : map (cons '/') rest
+makeURL :: SnapName -> [Text] -> Handler Text
+makeURL snapName rest = do
+    bucketUrl <- getsYesod (appDownloadBucketUrl . appSettings)
+    pure . concat
+        $ bucketUrl
+        : "/"
+        : toPathPiece snapName
+        : map (cons '/') rest
 
 shouldRedirect :: Bool
 shouldRedirect = False
@@ -27,7 +28,7 @@ getHaddockR snapName rest
         result <- redirectWithVersion snapName rest
         case result of
             Just route -> redirect route
-            Nothing -> redirect $ makeURL snapName rest
+            Nothing -> redirect =<< makeURL snapName rest
     | Just docType <- mdocType = do
         cacheSeconds $ 60 * 60 * 24 * 7
         result <- redirectWithVersion snapName rest
@@ -41,7 +42,7 @@ getHaddockR snapName rest
                             return ("text/html; charset=utf-8", mstyle /= Just "stackage")
                         DocJson ->
                             return ("application/jsontml; charset=utf-8", True)
-                req <- parseRequest $ unpack $ makeURL snapName rest
+                req <- parseRequest =<< unpack <$> makeURL snapName rest
                 man <- getHttpManager <$> getYesod
                 (_, res) <- runReaderT (acquireResponse req >>= allocateAcquire) man
                 if plain
@@ -54,7 +55,7 @@ getHaddockR snapName rest
                                 peekC >>= maybe (return ()) (const $ yield $ encodeUtf8 extra)
                                 mapC id) .|
                             mapC (Chunk . toBuilder)
-    | otherwise = redirect $ makeURL snapName rest
+    | otherwise = redirect =<< makeURL snapName rest
   where
     mdocType =
         case T.takeEnd 5 <$> headMay (reverse rest) of
@@ -141,6 +142,9 @@ getHaddockBackupR (snap':rest)
   | Just branch <- fromPathPiece snap' = track "Handler.Haddock.getHaddockBackupR" $ do
       snapName <- newestSnapshot branch >>= maybe notFound pure
       redirect $ HaddockR snapName rest
-getHaddockBackupR rest = track "Handler.Haddock.getHaddockBackupR" $ redirect $ concat
-    $ "https://s3.amazonaws.com/haddock.stackage.org"
-    : map (cons '/') rest
+getHaddockBackupR rest = track "Handler.Haddock.getHaddockBackupR" $ do
+    bucketUrl <- getsYesod (appDownloadBucketUrl . appSettings)
+    redirect
+        $ concat
+        $ bucketUrl
+        : map (cons '/') rest

src/Handler/MirrorStatus.hs

@@ -12,6 +12,8 @@ import RIO.Time (diffUTCTime, addUTCTime, getCurrentTime)
 import Text.XML.Stream.Parse
 import Data.XML.Types (Event (EventContent), Content (ContentText))
 import qualified Prelude
+import qualified Data.Aeson.Key as Aeson
+import qualified Data.Aeson.KeyMap as Aeson
 
 getMirrorStatusR :: Handler Html
 getMirrorStatusR = do
@@ -148,7 +150,7 @@ getLastModifiedGit org repo ref = do
 
 lookupJ :: MonadThrow m => Text -> Value -> m Value
 lookupJ key (Object o) =
-    case lookup key o of
+    case Aeson.lookup (Aeson.fromText key) o of
         Nothing -> error $ "Key not found: " ++ show key
         Just x -> return x
 lookupJ key val = error $ concat

src/Handler/StackageIndex.hs

@@ -2,13 +2,12 @@
 module Handler.StackageIndex where
 
 import Import
-import Stackage.Database.Types (haddockBucketName)
 
 getStackageIndexR :: SnapName -> Handler TypedContent
-getStackageIndexR slug =
+getStackageIndexR slug = do
+    bucketUrl <- getsYesod (appDownloadBucketUrl . appSettings)
     redirect $ concat
-        [ "https://s3.amazonaws.com/"
-        , haddockBucketName
+        [ bucketUrl
         , "/package-index/"
         , toPathPiece slug
         , ".tar.gz"

src/Settings.hs

@@ -56,6 +56,8 @@ data AppSettings = AppSettings
     -- ^ Force redirect to SSL
     , appDevDownload            :: Bool
     -- ^ Controls how Git and database resources are downloaded (True means less downloading)
+    , appDownloadBucketUrl      :: Text
+    -- ^ Publicly-accessible URL for the bucket holding Haddock contents.
     }
 
 data DatabaseSettings
@@ -109,6 +111,7 @@ instance FromJSON AppSettings where
         appSkipCombining          <- o .:? "skip-combining"   .!= dev
         appForceSsl               <- o .:? "force-ssl"        .!= not dev
         appDevDownload            <- o .:? "dev-download"     .!= dev
+        appDownloadBucketUrl      <- o .:? "download-bucket-url" .!= "https://s3.amazonaws.com/haddock.stackage.org"
 
         return AppSettings {..}