Migrate registry-docgen to registry-dev

.env.example

@@ -30,10 +30,12 @@ PACCHETTIBOTTI_ED25519_PUB="c3NoLWVkMjU1MTkgYWJjeHl6IHBhY2NoZXR0aWJvdHRpQHB1cmVz
 PACCHETTIBOTTI_ED25519="YWJjeHl6"
 
 # DigitalOcean Spaces credentials for S3-compatible storage
-# Used for: uploading/downloading package tarballs
+# Used for: uploading/downloading package tarballs and documentation artifacts
 SPACES_KEY="digitalocean_spaces_key"
 SPACES_SECRET="digitalocean_spaces_secret"
 
+# Separate bucket for replaceable, derived documentation artifacts
+DOCS_BUCKET="purescript-registry-docs"
 
 # -----------------------------------------------------------------------------
 # Debug / Development Options

CONTRIBUTING.md

@@ -27,6 +27,7 @@ spago test
 The registry is a significant PureScript application split into several runnable modules.
 
 - `app` is the main application and contains the registry server and the GitHub-based API. App code goes here.
+- `docgen` contains the registry's internal documentation model and codecs, legacy Pursuit conversion, re-export resolution, and HTML rendering. See the [`docgen` README](./docgen/README.md) for its architecture and ownership boundaries.
 - `foreign` contains library code for FFI bindings to JavaScript libraries. Any FFI you write should go here.
 - `lib` contains library code meant for other PureScript packages (such as Spago) to reuse. Core registry types and functions go here, and we are careful not to introduce breaking changes unless absolutely necessary.
 - `scripts` contains runnable modules written on top of the app for performing registry tasks like uploading and transferring packages.

LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2026 PureScript contributors
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

app-e2e/spago.yaml

@@ -19,6 +19,7 @@ package:
     - node-process
     - ordered-collections
     - registry-app
+    - registry-docgen
     - registry-foreign
     - registry-lib
     - registry-scripts

app-e2e/src/Test/E2E/DocsStorage.purs

@@ -0,0 +1,128 @@
+module Test.E2E.DocsStorage (spec) where
+
+import Registry.App.Prelude
+
+import Data.Array as Array
+import Data.Codec.JSON as CJ
+import Data.Map as Map
+import Data.String as String
+import Effect.Aff as Aff
+import JSON as JSON
+import Registry.App.Effect.DocsStorage (DOCS_STORAGE)
+import Registry.App.Effect.DocsStorage as DocsStorage
+import Registry.App.Effect.Env as Env
+import Registry.App.Effect.Log as Log
+import Registry.Docgen.Codec as Docgen.Codec
+import Registry.Docgen.Docs (DocPackage(..), SourceArtifact(..), schemaVersion)
+import Registry.License as License
+import Registry.PackageName (PackageName)
+import Registry.Test.Assert as Assert
+import Registry.Test.Fixtures (defaultHash, defaultLocation)
+import Registry.Test.Utils as Utils
+import Registry.Version (Version)
+import Run (AFF, EFFECT, Run)
+import Run as Run
+import Run.Except (EXCEPT)
+import Run.Except as Except
+import Test.E2E.Support.Env (E2E, E2ESpec)
+import Test.E2E.Support.WireMock as WireMock
+import Test.Spec as Spec
+
+spec :: E2ESpec
+spec = do
+  Spec.it "supports the complete documentation lifecycle through S3" do
+    runDocsStorage (DocsStorage.exists packageName packageVersion) >>= (_ `Assert.shouldEqual` false)
+
+    runDocsStorage $ DocsStorage.upload docs
+    runDocsStorage (DocsStorage.exists packageName packageVersion) >>= (_ `Assert.shouldEqual` true)
+
+    stored <- runDocsStorage $ DocsStorage.download packageName packageVersion
+    CJ.encode Docgen.Codec.docPackage stored `Assert.shouldEqual` CJ.encode Docgen.Codec.docPackage docs
+
+    duplicate <- runDocsStorageResult $ DocsStorage.upload docs
+    case duplicate of
+      Left error -> String.contains (String.Pattern "already exists") error `Assert.shouldEqual` true
+      Right _ -> Assert.fail "Immutable upload unexpectedly replaced documentation"
+
+    runDocsStorage $ DocsStorage.replace replacement
+    replaced <- runDocsStorage $ DocsStorage.download packageName packageVersion
+    CJ.encode Docgen.Codec.docPackage replaced `Assert.shouldEqual` CJ.encode Docgen.Codec.docPackage replacement
+
+    runDocsStorage $ DocsStorage.delete packageName packageVersion
+    runDocsStorage (DocsStorage.exists packageName packageVersion) >>= (_ `Assert.shouldEqual` false)
+    runDocsStorage $ DocsStorage.delete packageName packageVersion
+
+    requests <- WireMock.getStorageRequests
+    let objectRequests = WireMock.filterByUrlContaining ("/" <> DocsStorage.formatDocsPath packageName packageVersion) requests
+    let putRequests = WireMock.filterByMethod "PUT" objectRequests
+    let getRequests = WireMock.filterByMethod "GET" objectRequests
+    let deleteRequests = WireMock.filterByMethod "DELETE" objectRequests
+    Array.length putRequests `Assert.shouldEqual` 2
+    Array.length getRequests `Assert.shouldEqual` 2
+    Array.length deleteRequests `Assert.shouldEqual` 2
+    let putBodies = map decodeBody putRequests
+    Array.any (_ == CJ.encode Docgen.Codec.docPackage docs) putBodies `Assert.shouldEqual` true
+    Array.any (_ == CJ.encode Docgen.Codec.docPackage replacement) putBodies `Assert.shouldEqual` true
+
+runDocsStorage
+  :: forall a
+   . Run (DOCS_STORAGE + Env.RESOURCE_ENV + Log.LOG + EXCEPT String + AFF + EFFECT + ()) a
+  -> E2E a
+runDocsStorage operation = do
+  result <- runDocsStorageResult operation
+  case result of
+    Left error -> liftAff $ Aff.throwError $ Aff.error error
+    Right value -> pure value
+
+runDocsStorageResult
+  :: forall a
+   . Run (DOCS_STORAGE + Env.RESOURCE_ENV + Log.LOG + EXCEPT String + AFF + EFFECT + ()) a
+  -> E2E (Either String a)
+runDocsStorageResult operation = do
+  resourceEnv <- Env.lookupResourceEnv
+  key <- Env.lookupRequired Env.spacesKey
+  secret <- Env.lookupRequired Env.spacesSecret
+  bucket <- Env.lookupWithDefault Env.docsBucket "purescript-registry-docs"
+  liftAff $ operation
+    # Except.runExcept
+    # DocsStorage.interpret (DocsStorage.handleS3 { bucket, s3: { key, secret } })
+    # Env.runResourceEnv resourceEnv
+    # Log.interpret (\(Log.Log _ _ next) -> pure next)
+    # Run.runBaseAff'
+
+decodeBody :: WireMock.WireMockRequest -> JSON
+decodeBody request =
+  Utils.fromRight "S3 PUT request body was not JSON"
+    $ JSON.parse
+    $ Utils.fromJust "S3 PUT request did not contain a body" request.body
+
+docs :: DocPackage
+docs = DocPackage
+  { schemaVersion
+  , compilerVersion
+  , sourceArtifact: SourceArtifact { bytes: 42.0, hash: defaultHash }
+  , dependencies: Map.empty
+  , description: Nothing
+  , license: Utils.fromRight "license" $ License.parse "BSD-3-Clause"
+  , location: defaultLocation
+  , locationRef: Nothing
+  , modules: []
+  , name: packageName
+  , readme: Nothing
+  , resolvedDependencies: Map.empty
+  , resolvedModulePackages: Map.empty
+  , version: packageVersion
+  }
+
+replacement :: DocPackage
+replacement = case docs of
+  DocPackage package -> DocPackage package { description = Just "Replacement" }
+
+packageName :: PackageName
+packageName = Utils.unsafePackageName "docs-storage-test"
+
+packageVersion :: Version
+packageVersion = Utils.unsafeVersion "1.0.0"
+
+compilerVersion :: Version
+compilerVersion = Utils.unsafeVersion "0.15.15"

app-e2e/src/Test/Main.purs

@@ -3,6 +3,7 @@ module Test.E2E.Main (main) where
 import Registry.App.Prelude
 
 import Data.Time.Duration (Milliseconds(..))
+import Test.E2E.DocsStorage as DocsStorage
 import Test.E2E.Endpoint.Jobs as Jobs
 import Test.E2E.Endpoint.PackageSets as PackageSets
 import Test.E2E.Endpoint.Publish as Publish
@@ -33,6 +34,8 @@ main = do
         stashGitFixtures
 
     Spec.before_ resetTestState $ Spec.after_ assertReposClean $ Spec.describe "E2E Tests" do
+      Spec.describe "DocsStorage" DocsStorage.spec
+
       Spec.describe "Endpoints" do
         Spec.describe "Publish" Publish.spec
         Spec.describe "Jobs" Jobs.spec

app/spago.yaml

@@ -58,6 +58,7 @@ package:
     - profunctor
     - record
     - refs
+    - registry-docgen
     - registry-foreign
     - registry-lib
     - run

app/src/App/API.purs

@@ -74,15 +74,15 @@ import Registry.App.Effect.Log (LOG)
 import Registry.App.Effect.Log as Log
 import Registry.App.Effect.PackageSets (Change(..), PACKAGE_SETS)
 import Registry.App.Effect.PackageSets as PackageSets
+import Registry.App.Effect.PackageStorage (PACKAGE_STORAGE)
+import Registry.App.Effect.PackageStorage as PackageStorage
 import Registry.App.Effect.Pursuit (PURSUIT)
 import Registry.App.Effect.Pursuit as Pursuit
 import Registry.App.Effect.Registry (REGISTRY, REGISTRY_READ)
 import Registry.App.Effect.Registry as ManifestIndex
 import Registry.App.Effect.Registry as Registry
 import Registry.App.Effect.Source (SOURCE)
 import Registry.App.Effect.Source as Source
-import Registry.App.Effect.Storage (STORAGE)
-import Registry.App.Effect.Storage as Storage
 import Registry.App.Legacy.Manifest as Legacy.Manifest
 import Registry.App.Legacy.Types (RawPackageName(..), rawPackageNameMapCodec)
 import Registry.App.Manifest.SpagoYaml as SpagoYaml
@@ -242,7 +242,7 @@ packageSetUpdate details = do
       Registry.mirrorPackageSet packageSet
       Log.notice "Mirrored a new legacy package set."
 
-type AuthenticatedEffects r = (REGISTRY + STORAGE + GITHUB + PACCHETTIBOTTI_ENV + LOG + EXCEPT String + AFF + EFFECT + r)
+type AuthenticatedEffects r = (REGISTRY + PACKAGE_STORAGE + GITHUB + PACCHETTIBOTTI_ENV + LOG + EXCEPT String + AFF + EFFECT + r)
 
 -- | Run an authenticated package operation, ie. an unpublish or a transfer.
 authenticated :: forall r. AuthenticatedData -> Run (AuthenticatedEffects + r) Unit
@@ -304,7 +304,7 @@ authenticated auth = case auth.payload of
         -- violations before performing any irreversible side effects like deleting
         -- the tarball from storage.
         Registry.deleteManifest payload.name payload.version
-        Storage.delete payload.name payload.version
+        PackageStorage.delete payload.name payload.version
         Registry.writeMetadata payload.name updated
         Log.notice $ "Unpublished " <> formatted <> "!"
 
@@ -339,7 +339,7 @@ authenticated auth = case auth.payload of
         Registry.writeMetadata payload.name updated
         Log.notice "Successfully transferred your package!"
 
-type PublishEffects r = (RESOURCE_ENV + PURSUIT + REGISTRY + STORAGE + SOURCE + GITHUB + COMPILER_CACHE + PURS_GRAPH_CACHE + LOG + EXCEPT String + AFF + EFFECT + r)
+type PublishEffects r = (RESOURCE_ENV + PURSUIT + REGISTRY + PACKAGE_STORAGE + SOURCE + GITHUB + COMPILER_CACHE + PURS_GRAPH_CACHE + LOG + EXCEPT String + AFF + EFFECT + r)
 
 -- | Resolve both compiler and resolutions for a publish operation.
 -- | Will come up with some sort of plan if not provided with a compiler and/or resolutions.
@@ -734,7 +734,7 @@ publish payload = do
     reconcileExistingPublication info = do
       let storedPackageDirname = PackageName.print receivedManifest.name <> "-" <> Version.print receivedManifest.version
       let storedTarballPath = Path.concat [ tmp, "stored-" <> storedPackageDirname <> ".tar.gz" ]
-      Storage.download receivedManifest.name receivedManifest.version storedTarballPath { hash: info.hash, bytes: info.bytes }
+      PackageStorage.download receivedManifest.name receivedManifest.version storedTarballPath { hash: info.hash, bytes: info.bytes }
       when (isNothing existingManifest) do
         Tar.extract { cwd: tmp, archive: storedTarballPath }
         storedManifest <- Run.liftAff (readJsonFile Manifest.codec (Path.concat [ tmp, storedPackageDirname, "purs.json" ])) >>= case _ of
@@ -965,13 +965,13 @@ publish payload = do
       Log.info $ "Tarball size of " <> show bytes <> " bytes is acceptable."
       Log.info $ "Tarball hash: " <> Sha256.print hash
 
-      Except.runExcept (Storage.upload receivedManifest.name receivedManifest.version tarballPath) >>= case _ of
+      Except.runExcept (PackageStorage.upload receivedManifest.name receivedManifest.version tarballPath) >>= case _ of
         Right _ -> pure unit
         Left uploadError -> do
-          Except.runExcept (Storage.query receivedManifest.name) >>= case _ of
+          Except.runExcept (PackageStorage.query receivedManifest.name) >>= case _ of
             Right storedVersions | Set.member receivedManifest.version storedVersions -> do
               let storedTarballPath = tarballPath <> ".stored"
-              Except.runExcept (Storage.download receivedManifest.name receivedManifest.version storedTarballPath { hash, bytes }) >>= case _ of
+              Except.runExcept (PackageStorage.download receivedManifest.name receivedManifest.version storedTarballPath { hash, bytes }) >>= case _ of
                 Left error ->
                   Except.throw $ "Cannot resume publishing " <> formatPackageVersion receivedManifest.name receivedManifest.version <> " because the existing tarball in storage could not be verified against the package source: " <> error
                 Right _ ->
@@ -1067,7 +1067,7 @@ type FindAllCompilersResult =
 findAllCompilers
   :: forall r
    . { source :: FilePath, manifest :: Manifest, compilers :: NonEmptyArray Version }
-  -> Run (REGISTRY_READ + STORAGE + COMPILER_CACHE + LOG + AFF + EFFECT + EXCEPT String + r) FindAllCompilersResult
+  -> Run (REGISTRY_READ + PACKAGE_STORAGE + COMPILER_CACHE + LOG + AFF + EFFECT + EXCEPT String + r) FindAllCompilersResult
 findAllCompilers { source, manifest, compilers } = do
   compilerIndex <- MatrixBuilder.readCompilerIndex
   checkedCompilers <- for compilers \target -> do