From 73a960d9f991a51688040fd18cc26d3dc0148326 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Tue, 28 Jul 2026 02:40:38 +0000 Subject: [PATCH 1/8] Add registry-owned documentation artifacts Move the docgen implementation into the monorepo, define and verify the canonical documentation format, and add separate replaceable documentation storage with filesystem and S3 coverage. Rename package tarball storage for clarity and add historical compatibility verification. --- .env.example | 4 +- CONTRIBUTING.md | 1 + app-e2e/spago.yaml | 1 + app-e2e/src/Test/E2E/DocsStorage.purs | 128 ++++ app-e2e/src/Test/Main.purs | 3 + app/spago.yaml | 1 + app/src/App/API.purs | 20 +- app/src/App/Effect/DocsStorage.purs | 212 +++++++ app/src/App/Effect/Env.purs | 4 + app/src/App/Effect/PackageSets.purs | 8 +- .../{Storage.purs => PackageStorage.purs} | 96 +-- app/src/App/Server/Env.purs | 20 +- app/src/App/Server/MatrixBuilder.purs | 10 +- app/test/App/API.purs | 16 +- app/test/App/Effect/DocsStorage.purs | 109 ++++ app/test/Main.purs | 2 + app/test/Test/Assert/Run.purs | 38 +- docgen/AGENTS.md | 25 + docgen/README.md | 65 +++ docgen/fixtures/undefined-1.0.2.json | 1 + docgen/package.json | 9 + docgen/spago.yaml | 39 ++ docgen/src/Registry/Docgen/Codec.purs | 505 ++++++++++++++++ docgen/src/Registry/Docgen/Commonmark.js | 46 ++ docgen/src/Registry/Docgen/Commonmark.purs | 14 + docgen/src/Registry/Docgen/Convert.purs | 337 +++++++++++ docgen/src/Registry/Docgen/Decoder.purs | 145 +++++ docgen/src/Registry/Docgen/Docs.purs | 335 +++++++++++ docgen/src/Registry/Docgen/Generate.purs | 137 +++++ docgen/src/Registry/Docgen/HTML.purs | 196 +++++++ docgen/src/Registry/Docgen/Legacy/Docs.purs | 180 ++++++ docgen/src/Registry/Docgen/Legacy/JSON.purs | 512 ++++++++++++++++ .../src/Registry/Docgen/Package/Render.purs | 550 ++++++++++++++++++ .../Registry/Docgen/Package/Render/Code.purs | 335 +++++++++++ docgen/src/Registry/Docgen/Package/Types.purs | 35 ++ docgen/src/Registry/Docgen/Reexports.purs | 428 ++++++++++++++ docgen/test/Test/Registry/Docgen.purs | 495 ++++++++++++++++ flake.nix | 2 + foreign/src/Foreign/S3.js | 7 + foreign/src/Foreign/S3.purs | 17 + nix/overlay.nix | 8 +- nix/test/config.nix | 130 ++++- nix/test/integration.nix | 3 +- package-lock.json | 47 ++ package.json | 1 + scripts/spago.yaml | 1 + scripts/src/VerifyIntegrity.purs | 6 +- scripts/src/VerifyLegacyDocs.purs | 104 ++++ spago.lock | 61 ++ 49 files changed, 5341 insertions(+), 108 deletions(-) create mode 100644 app-e2e/src/Test/E2E/DocsStorage.purs create mode 100644 app/src/App/Effect/DocsStorage.purs rename app/src/App/Effect/{Storage.purs => PackageStorage.purs} (82%) create mode 100644 app/test/App/Effect/DocsStorage.purs create mode 100644 docgen/AGENTS.md create mode 100644 docgen/README.md create mode 100644 docgen/fixtures/undefined-1.0.2.json create mode 100644 docgen/package.json create mode 100644 docgen/spago.yaml create mode 100644 docgen/src/Registry/Docgen/Codec.purs create mode 100644 docgen/src/Registry/Docgen/Commonmark.js create mode 100644 docgen/src/Registry/Docgen/Commonmark.purs create mode 100644 docgen/src/Registry/Docgen/Convert.purs create mode 100644 docgen/src/Registry/Docgen/Decoder.purs create mode 100644 docgen/src/Registry/Docgen/Docs.purs create mode 100644 docgen/src/Registry/Docgen/Generate.purs create mode 100644 docgen/src/Registry/Docgen/HTML.purs create mode 100644 docgen/src/Registry/Docgen/Legacy/Docs.purs create mode 100644 docgen/src/Registry/Docgen/Legacy/JSON.purs create mode 100644 docgen/src/Registry/Docgen/Package/Render.purs create mode 100644 docgen/src/Registry/Docgen/Package/Render/Code.purs create mode 100644 docgen/src/Registry/Docgen/Package/Types.purs create mode 100644 docgen/src/Registry/Docgen/Reexports.purs create mode 100644 docgen/test/Test/Registry/Docgen.purs create mode 100644 scripts/src/VerifyLegacyDocs.purs diff --git a/.env.example b/.env.example index ea0488ffe..492cfc31e 100644 --- a/.env.example +++ b/.env.example @@ -30,10 +30,12 @@ PACCHETTIBOTTI_ED25519_PUB="c3NoLWVkMjU1MTkgYWJjeHl6IHBhY2NoZXR0aWJvdHRpQHB1cmVz PACCHETTIBOTTI_ED25519="YWJjeHl6" # DigitalOcean Spaces credentials for S3-compatible storage -# Used for: uploading/downloading package tarballs +# Used for: uploading/downloading package tarballs and documentation artifacts SPACES_KEY="digitalocean_spaces_key" SPACES_SECRET="digitalocean_spaces_secret" +# Separate bucket for replaceable, derived documentation artifacts +DOCS_BUCKET="purescript-registry-docs" # ----------------------------------------------------------------------------- # Debug / Development Options diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef018a3fd..acb38a423 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,7 @@ spago test The registry is a significant PureScript application split into several runnable modules. - `app` is the main application and contains the registry server and the GitHub-based API. App code goes here. +- `docgen` contains the registry's internal documentation model and codecs, legacy Pursuit conversion, re-export resolution, and HTML rendering. See the [`docgen` README](./docgen/README.md) for its architecture and ownership boundaries. - `foreign` contains library code for FFI bindings to JavaScript libraries. Any FFI you write should go here. - `lib` contains library code meant for other PureScript packages (such as Spago) to reuse. Core registry types and functions go here, and we are careful not to introduce breaking changes unless absolutely necessary. - `scripts` contains runnable modules written on top of the app for performing registry tasks like uploading and transferring packages. diff --git a/app-e2e/spago.yaml b/app-e2e/spago.yaml index 90002234f..993d7ed8a 100644 --- a/app-e2e/spago.yaml +++ b/app-e2e/spago.yaml @@ -19,6 +19,7 @@ package: - node-process - ordered-collections - registry-app + - registry-docgen - registry-foreign - registry-lib - registry-scripts diff --git a/app-e2e/src/Test/E2E/DocsStorage.purs b/app-e2e/src/Test/E2E/DocsStorage.purs new file mode 100644 index 000000000..4e516fd2a --- /dev/null +++ b/app-e2e/src/Test/E2E/DocsStorage.purs @@ -0,0 +1,128 @@ +module Test.E2E.DocsStorage (spec) where + +import Registry.App.Prelude + +import Data.Array as Array +import Data.Codec.JSON as CJ +import Data.Map as Map +import Data.String as String +import Effect.Aff as Aff +import JSON as JSON +import Registry.App.Effect.DocsStorage (DOCS_STORAGE) +import Registry.App.Effect.DocsStorage as DocsStorage +import Registry.App.Effect.Env as Env +import Registry.App.Effect.Log as Log +import Registry.Docgen.Codec as Docgen.Codec +import Registry.Docgen.Docs (DocPackage(..), SourceArtifact(..), schemaVersion) +import Registry.License as License +import Registry.PackageName (PackageName) +import Registry.Test.Assert as Assert +import Registry.Test.Fixtures (defaultHash, defaultLocation) +import Registry.Test.Utils as Utils +import Registry.Version (Version) +import Run (AFF, EFFECT, Run) +import Run as Run +import Run.Except (EXCEPT) +import Run.Except as Except +import Test.E2E.Support.Env (E2E, E2ESpec) +import Test.E2E.Support.WireMock as WireMock +import Test.Spec as Spec + +spec :: E2ESpec +spec = do + Spec.it "supports the complete documentation lifecycle through S3" do + runDocsStorage (DocsStorage.exists packageName packageVersion) >>= (_ `Assert.shouldEqual` false) + + runDocsStorage $ DocsStorage.upload docs + runDocsStorage (DocsStorage.exists packageName packageVersion) >>= (_ `Assert.shouldEqual` true) + + stored <- runDocsStorage $ DocsStorage.download packageName packageVersion + CJ.encode Docgen.Codec.docPackage stored `Assert.shouldEqual` CJ.encode Docgen.Codec.docPackage docs + + duplicate <- runDocsStorageResult $ DocsStorage.upload docs + case duplicate of + Left error -> String.contains (String.Pattern "already exists") error `Assert.shouldEqual` true + Right _ -> Assert.fail "Immutable upload unexpectedly replaced documentation" + + runDocsStorage $ DocsStorage.replace replacement + replaced <- runDocsStorage $ DocsStorage.download packageName packageVersion + CJ.encode Docgen.Codec.docPackage replaced `Assert.shouldEqual` CJ.encode Docgen.Codec.docPackage replacement + + runDocsStorage $ DocsStorage.delete packageName packageVersion + runDocsStorage (DocsStorage.exists packageName packageVersion) >>= (_ `Assert.shouldEqual` false) + runDocsStorage $ DocsStorage.delete packageName packageVersion + + requests <- WireMock.getStorageRequests + let objectRequests = WireMock.filterByUrlContaining ("/" <> DocsStorage.formatDocsPath packageName packageVersion) requests + let putRequests = WireMock.filterByMethod "PUT" objectRequests + let getRequests = WireMock.filterByMethod "GET" objectRequests + let deleteRequests = WireMock.filterByMethod "DELETE" objectRequests + Array.length putRequests `Assert.shouldEqual` 2 + Array.length getRequests `Assert.shouldEqual` 2 + Array.length deleteRequests `Assert.shouldEqual` 2 + let putBodies = map decodeBody putRequests + Array.any (_ == CJ.encode Docgen.Codec.docPackage docs) putBodies `Assert.shouldEqual` true + Array.any (_ == CJ.encode Docgen.Codec.docPackage replacement) putBodies `Assert.shouldEqual` true + +runDocsStorage + :: forall a + . Run (DOCS_STORAGE + Env.RESOURCE_ENV + Log.LOG + EXCEPT String + AFF + EFFECT + ()) a + -> E2E a +runDocsStorage operation = do + result <- runDocsStorageResult operation + case result of + Left error -> liftAff $ Aff.throwError $ Aff.error error + Right value -> pure value + +runDocsStorageResult + :: forall a + . Run (DOCS_STORAGE + Env.RESOURCE_ENV + Log.LOG + EXCEPT String + AFF + EFFECT + ()) a + -> E2E (Either String a) +runDocsStorageResult operation = do + resourceEnv <- Env.lookupResourceEnv + key <- Env.lookupRequired Env.spacesKey + secret <- Env.lookupRequired Env.spacesSecret + bucket <- Env.lookupWithDefault Env.docsBucket "purescript-registry-docs" + liftAff $ operation + # Except.runExcept + # DocsStorage.interpret (DocsStorage.handleS3 { bucket, s3: { key, secret } }) + # Env.runResourceEnv resourceEnv + # Log.interpret (\(Log.Log _ _ next) -> pure next) + # Run.runBaseAff' + +decodeBody :: WireMock.WireMockRequest -> JSON +decodeBody request = + Utils.fromRight "S3 PUT request body was not JSON" + $ JSON.parse + $ Utils.fromJust "S3 PUT request did not contain a body" request.body + +docs :: DocPackage +docs = DocPackage + { schemaVersion + , compilerVersion + , sourceArtifact: SourceArtifact { bytes: 42.0, hash: defaultHash } + , dependencies: Map.empty + , description: Nothing + , license: Utils.fromRight "license" $ License.parse "BSD-3-Clause" + , location: defaultLocation + , locationRef: Nothing + , modules: [] + , name: packageName + , readme: Nothing + , resolvedDependencies: Map.empty + , resolvedModulePackages: Map.empty + , version: packageVersion + } + +replacement :: DocPackage +replacement = case docs of + DocPackage package -> DocPackage package { description = Just "Replacement" } + +packageName :: PackageName +packageName = Utils.unsafePackageName "docs-storage-test" + +packageVersion :: Version +packageVersion = Utils.unsafeVersion "1.0.0" + +compilerVersion :: Version +compilerVersion = Utils.unsafeVersion "0.15.15" diff --git a/app-e2e/src/Test/Main.purs b/app-e2e/src/Test/Main.purs index a0f88464f..6a1c76bf0 100644 --- a/app-e2e/src/Test/Main.purs +++ b/app-e2e/src/Test/Main.purs @@ -3,6 +3,7 @@ module Test.E2E.Main (main) where import Registry.App.Prelude import Data.Time.Duration (Milliseconds(..)) +import Test.E2E.DocsStorage as DocsStorage import Test.E2E.Endpoint.Jobs as Jobs import Test.E2E.Endpoint.PackageSets as PackageSets import Test.E2E.Endpoint.Publish as Publish @@ -33,6 +34,8 @@ main = do stashGitFixtures Spec.before_ resetTestState $ Spec.after_ assertReposClean $ Spec.describe "E2E Tests" do + Spec.describe "DocsStorage" DocsStorage.spec + Spec.describe "Endpoints" do Spec.describe "Publish" Publish.spec Spec.describe "Jobs" Jobs.spec diff --git a/app/spago.yaml b/app/spago.yaml index 166ab944f..4ee014f64 100644 --- a/app/spago.yaml +++ b/app/spago.yaml @@ -58,6 +58,7 @@ package: - profunctor - record - refs + - registry-docgen - registry-foreign - registry-lib - run diff --git a/app/src/App/API.purs b/app/src/App/API.purs index a4b69c5d3..d0eb1549d 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -74,6 +74,8 @@ import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log import Registry.App.Effect.PackageSets (Change(..), PACKAGE_SETS) import Registry.App.Effect.PackageSets as PackageSets +import Registry.App.Effect.PackageStorage (PACKAGE_STORAGE) +import Registry.App.Effect.PackageStorage as PackageStorage import Registry.App.Effect.Pursuit (PURSUIT) import Registry.App.Effect.Pursuit as Pursuit import Registry.App.Effect.Registry (REGISTRY, REGISTRY_READ) @@ -81,8 +83,6 @@ import Registry.App.Effect.Registry as ManifestIndex import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (SOURCE) import Registry.App.Effect.Source as Source -import Registry.App.Effect.Storage (STORAGE) -import Registry.App.Effect.Storage as Storage import Registry.App.Legacy.Manifest as Legacy.Manifest import Registry.App.Legacy.Types (RawPackageName(..), rawPackageNameMapCodec) import Registry.App.Manifest.SpagoYaml as SpagoYaml @@ -242,7 +242,7 @@ packageSetUpdate details = do Registry.mirrorPackageSet packageSet Log.notice "Mirrored a new legacy package set." -type AuthenticatedEffects r = (REGISTRY + STORAGE + GITHUB + PACCHETTIBOTTI_ENV + LOG + EXCEPT String + AFF + EFFECT + r) +type AuthenticatedEffects r = (REGISTRY + PACKAGE_STORAGE + GITHUB + PACCHETTIBOTTI_ENV + LOG + EXCEPT String + AFF + EFFECT + r) -- | Run an authenticated package operation, ie. an unpublish or a transfer. authenticated :: forall r. AuthenticatedData -> Run (AuthenticatedEffects + r) Unit @@ -304,7 +304,7 @@ authenticated auth = case auth.payload of -- violations before performing any irreversible side effects like deleting -- the tarball from storage. Registry.deleteManifest payload.name payload.version - Storage.delete payload.name payload.version + PackageStorage.delete payload.name payload.version Registry.writeMetadata payload.name updated Log.notice $ "Unpublished " <> formatted <> "!" @@ -339,7 +339,7 @@ authenticated auth = case auth.payload of Registry.writeMetadata payload.name updated Log.notice "Successfully transferred your package!" -type PublishEffects r = (RESOURCE_ENV + PURSUIT + REGISTRY + STORAGE + SOURCE + GITHUB + COMPILER_CACHE + PURS_GRAPH_CACHE + LOG + EXCEPT String + AFF + EFFECT + r) +type PublishEffects r = (RESOURCE_ENV + PURSUIT + REGISTRY + PACKAGE_STORAGE + SOURCE + GITHUB + COMPILER_CACHE + PURS_GRAPH_CACHE + LOG + EXCEPT String + AFF + EFFECT + r) -- | Resolve both compiler and resolutions for a publish operation. -- | Will come up with some sort of plan if not provided with a compiler and/or resolutions. @@ -734,7 +734,7 @@ publish payload = do reconcileExistingPublication info = do let storedPackageDirname = PackageName.print receivedManifest.name <> "-" <> Version.print receivedManifest.version let storedTarballPath = Path.concat [ tmp, "stored-" <> storedPackageDirname <> ".tar.gz" ] - Storage.download receivedManifest.name receivedManifest.version storedTarballPath { hash: info.hash, bytes: info.bytes } + PackageStorage.download receivedManifest.name receivedManifest.version storedTarballPath { hash: info.hash, bytes: info.bytes } when (isNothing existingManifest) do Tar.extract { cwd: tmp, archive: storedTarballPath } storedManifest <- Run.liftAff (readJsonFile Manifest.codec (Path.concat [ tmp, storedPackageDirname, "purs.json" ])) >>= case _ of @@ -965,13 +965,13 @@ publish payload = do Log.info $ "Tarball size of " <> show bytes <> " bytes is acceptable." Log.info $ "Tarball hash: " <> Sha256.print hash - Except.runExcept (Storage.upload receivedManifest.name receivedManifest.version tarballPath) >>= case _ of + Except.runExcept (PackageStorage.upload receivedManifest.name receivedManifest.version tarballPath) >>= case _ of Right _ -> pure unit Left uploadError -> do - Except.runExcept (Storage.query receivedManifest.name) >>= case _ of + Except.runExcept (PackageStorage.query receivedManifest.name) >>= case _ of Right storedVersions | Set.member receivedManifest.version storedVersions -> do let storedTarballPath = tarballPath <> ".stored" - Except.runExcept (Storage.download receivedManifest.name receivedManifest.version storedTarballPath { hash, bytes }) >>= case _ of + Except.runExcept (PackageStorage.download receivedManifest.name receivedManifest.version storedTarballPath { hash, bytes }) >>= case _ of Left error -> Except.throw $ "Cannot resume publishing " <> formatPackageVersion receivedManifest.name receivedManifest.version <> " because the existing tarball in storage could not be verified against the package source: " <> error Right _ -> @@ -1067,7 +1067,7 @@ type FindAllCompilersResult = findAllCompilers :: forall r . { source :: FilePath, manifest :: Manifest, compilers :: NonEmptyArray Version } - -> Run (REGISTRY_READ + STORAGE + COMPILER_CACHE + LOG + AFF + EFFECT + EXCEPT String + r) FindAllCompilersResult + -> Run (REGISTRY_READ + PACKAGE_STORAGE + COMPILER_CACHE + LOG + AFF + EFFECT + EXCEPT String + r) FindAllCompilersResult findAllCompilers { source, manifest, compilers } = do compilerIndex <- MatrixBuilder.readCompilerIndex checkedCompilers <- for compilers \target -> do diff --git a/app/src/App/Effect/DocsStorage.purs b/app/src/App/Effect/DocsStorage.purs new file mode 100644 index 000000000..c6548bbb2 --- /dev/null +++ b/app/src/App/Effect/DocsStorage.purs @@ -0,0 +1,212 @@ +-- | Storage for canonical package documentation artifacts. Unlike package +-- | tarballs, docs are derived data and can be replaced after generator fixes. +module Registry.App.Effect.DocsStorage + ( DOCS_STORAGE + , DocsStorage(..) + , S3Env + , _docsStorage + , delete + , download + , exists + , formatDocsPath + , handleFs + , handleReadOnly + , handleS3 + , interpret + , replace + , upload + ) where + +import Registry.App.Prelude + +import Codec.JSON.DecodeError as DecodeError +import Data.Array as Array +import Effect.Aff as Aff +import Node.Buffer as Buffer +import Node.FS.Sync as FS.Sync +import Node.Path as Path +import Registry.App.Effect.Env (RESOURCE_ENV) +import Registry.App.Effect.Env as Env +import Registry.App.Effect.Log (LOG) +import Registry.App.Effect.Log as Log +import Registry.Docgen.Codec as Docgen.Codec +import Registry.Docgen.Docs (DocPackage(..)) +import Registry.Foreign.FSExtra as FS.Extra +import Registry.Foreign.S3 as S3 +import Registry.PackageName (PackageName) +import Registry.PackageName as PackageName +import Registry.Version (Version) +import Registry.Version as Version +import Run (AFF, EFFECT, Run) +import Run as Run +import Run.Except (EXCEPT) +import Run.Except as Except + +data DocsStorage a + = Upload DocPackage (Either String Unit -> a) + | Replace DocPackage (Either String Unit -> a) + | Download PackageName Version (Either String DocPackage -> a) + | Exists PackageName Version (Either String Boolean -> a) + | Delete PackageName Version (Either String Unit -> a) + +derive instance Functor DocsStorage + +type DOCS_STORAGE r = (docsStorage :: DocsStorage | r) + +_docsStorage :: Proxy "docsStorage" +_docsStorage = Proxy + +upload :: forall r. DocPackage -> Run (DOCS_STORAGE + EXCEPT String + r) Unit +upload docs = Except.rethrow =<< Run.lift _docsStorage (Upload docs identity) + +replace :: forall r. DocPackage -> Run (DOCS_STORAGE + EXCEPT String + r) Unit +replace docs = Except.rethrow =<< Run.lift _docsStorage (Replace docs identity) + +download :: forall r. PackageName -> Version -> Run (DOCS_STORAGE + EXCEPT String + r) DocPackage +download name version = Except.rethrow =<< Run.lift _docsStorage (Download name version identity) + +exists :: forall r. PackageName -> Version -> Run (DOCS_STORAGE + EXCEPT String + r) Boolean +exists name version = Except.rethrow =<< Run.lift _docsStorage (Exists name version identity) + +delete :: forall r. PackageName -> Version -> Run (DOCS_STORAGE + EXCEPT String + r) Unit +delete name version = Except.rethrow =<< Run.lift _docsStorage (Delete name version identity) + +interpret :: forall r a. (DocsStorage ~> Run r) -> Run (DOCS_STORAGE + r) a -> Run r a +interpret handler = Run.interpret (Run.on _docsStorage handler Run.send) + +formatDocsPath :: PackageName -> Version -> String +formatDocsPath name version = PackageName.print name <> "/" <> Version.print version <> ".json" + +handleFs :: forall r a. FilePath -> DocsStorage a -> Run (AFF + EFFECT + r) a +handleFs root = case _ of + Upload docs reply -> do + let { name, version } = packageIdentity docs + let path = Path.concat [ root, formatDocsPath name version ] + present <- Run.liftEffect $ FS.Sync.exists path + if present then + pure $ reply $ Left $ "Documentation for " <> formatPackageVersion name version <> " already exists." + else + reply <$> writeDocsFile path docs + + Replace docs reply -> do + let { name, version } = packageIdentity docs + reply <$> writeDocsFile (Path.concat [ root, formatDocsPath name version ]) docs + + Download name version reply -> do + let path = Path.concat [ root, formatDocsPath name version ] + result <- Run.liftAff $ readJsonFile Docgen.Codec.docPackage path + pure $ reply $ result + # lmap (\error -> "Could not read documentation for " <> formatPackageVersion name version <> ": " <> error) + >>= validateIdentity name version + + Exists name version reply -> do + present <- Run.liftEffect $ FS.Sync.exists $ Path.concat [ root, formatDocsPath name version ] + pure $ reply $ Right present + + Delete name version reply -> do + let path = Path.concat [ root, formatDocsPath name version ] + present <- Run.liftEffect $ FS.Sync.exists path + result <- if present then Run.liftAff $ Aff.attempt $ FS.Extra.remove path else pure $ Right unit + pure $ reply $ lmap (\error -> "Could not delete documentation for " <> formatPackageVersion name version <> ": " <> Aff.message error) result + where + writeDocsFile path docs = map (lmap Aff.message) $ Run.liftAff $ Aff.attempt do + FS.Extra.ensureDirectory $ Path.dirname path + writeJsonFile Docgen.Codec.docPackage path docs + +type S3Env = + { bucket :: String + , s3 :: S3.SpaceKey + } + +handleS3 :: forall r a. S3Env -> DocsStorage a -> Run (RESOURCE_ENV + LOG + AFF + EFFECT + r) a +handleS3 env = case _ of + Upload docs reply -> map (map reply) Except.runExcept do + let { name, version } = packageIdentity docs + s3 <- connectS3 env + whenM (objectExists s3 name version) do + Except.throw $ "Documentation for " <> formatPackageVersion name version <> " already exists." + putDocs s3 docs + + Replace docs reply -> map (map reply) Except.runExcept do + s3 <- connectS3 env + putDocs s3 docs + + Download name version reply -> map (map reply) Except.runExcept do + s3 <- connectS3 env + let key = formatDocsPath name version + result <- Run.liftAff $ withRetryOnTimeout $ Aff.attempt $ S3.getObject s3 { key } + buffer <- handleS3Result ("download documentation for " <> formatPackageVersion name version) result + contents <- Run.liftEffect $ Buffer.toString UTF8 buffer + docs <- Except.rethrow $ lmap DecodeError.print $ parseJson Docgen.Codec.docPackage contents + Except.rethrow $ validateIdentity name version docs + + Exists name version reply -> map (map reply) Except.runExcept do + s3 <- connectS3 env + objectExists s3 name version + + Delete name version reply -> map (map reply) Except.runExcept do + s3 <- connectS3 env + let key = formatDocsPath name version + result <- Run.liftAff $ withRetryOnTimeout $ Aff.attempt $ S3.deleteObject s3 { key } + void $ handleS3Result ("delete documentation for " <> formatPackageVersion name version) result + +handleReadOnly :: forall r a. S3Env -> DocsStorage a -> Run (RESOURCE_ENV + LOG + AFF + EFFECT + r) a +handleReadOnly env operation = case operation of + Upload docs reply -> do + let { name, version } = packageIdentity docs + Log.warn $ "Skipping documentation upload for " <> formatPackageVersion name version <> " in read-only mode." + pure $ reply $ Right unit + Replace docs reply -> do + let { name, version } = packageIdentity docs + Log.warn $ "Skipping documentation replacement for " <> formatPackageVersion name version <> " in read-only mode." + pure $ reply $ Right unit + Delete name version reply -> do + Log.warn $ "Skipping documentation deletion for " <> formatPackageVersion name version <> " in read-only mode." + pure $ reply $ Right unit + read -> handleS3 env read + +packageIdentity :: DocPackage -> { name :: PackageName, version :: Version } +packageIdentity (DocPackage { name, version }) = { name, version } + +validateIdentity :: PackageName -> Version -> DocPackage -> Either String DocPackage +validateIdentity expectedName expectedVersion docs = do + let actual = packageIdentity docs + if actual.name == expectedName && actual.version == expectedVersion then + Right docs + else + Left $ "Stored documentation at " <> formatDocsPath expectedName expectedVersion + <> " identifies itself as " + <> formatPackageVersion actual.name actual.version + <> "." + +connectS3 :: forall r. S3Env -> Run (RESOURCE_ENV + LOG + EXCEPT String + AFF + r) S3.Space +connectS3 env = do + { s3BucketUrl } <- Env.askResourceEnv + Log.debug $ "Connecting to documentation bucket " <> env.bucket <> "." + result <- Run.liftAff $ withRetryOnTimeout $ Aff.attempt $ S3.connect env.s3 s3BucketUrl env.bucket + handleS3Result "connect to the documentation storage backend" result + +objectExists :: forall r. S3.Space -> PackageName -> Version -> Run (LOG + EXCEPT String + AFF + r) Boolean +objectExists s3 name version = do + let key = formatDocsPath name version + result <- Run.liftAff $ withRetryOnTimeout $ Aff.attempt $ S3.listObjects s3 { prefix: key } + objects <- handleS3Result ("check documentation for " <> formatPackageVersion name version) result + pure $ Array.any (_.key >>> eq key) objects + +putDocs :: forall r. S3.Space -> DocPackage -> Run (LOG + EXCEPT String + AFF + EFFECT + r) Unit +putDocs s3 docs = do + let { name, version } = packageIdentity docs + let key = formatDocsPath name version + buffer <- Run.liftEffect $ Buffer.fromString (stringifyJson Docgen.Codec.docPackage docs) UTF8 + result <- Run.liftAff $ withRetryOnTimeout $ Aff.attempt $ S3.putObject s3 { key, body: buffer, acl: S3.PublicRead } + void $ handleS3Result ("upload documentation for " <> formatPackageVersion name version) result + +handleS3Result :: forall value r. String -> RetryResult Aff.Error value -> Run (LOG + EXCEPT String + r) value +handleS3Result operation = case _ of + Cancelled -> do + Log.error $ "Timed out while attempting to " <> operation <> "." + Except.throw $ "Could not " <> operation <> "." + Failed error -> do + Log.error $ "Failed to " <> operation <> ": " <> Aff.message error + Except.throw $ "Could not " <> operation <> "." + Succeeded value -> pure value diff --git a/app/src/App/Effect/Env.purs b/app/src/App/Effect/Env.purs index 268ac4f38..77054526a 100644 --- a/app/src/App/Effect/Env.purs +++ b/app/src/App/Effect/Env.purs @@ -178,6 +178,10 @@ spacesKey = EnvKey { key: "SPACES_KEY", decode: pure } spacesSecret :: EnvKey String spacesSecret = EnvKey { key: "SPACES_SECRET", decode: pure } +-- | The separate DigitalOcean Spaces bucket used for derived documentation. +docsBucket :: EnvKey String +docsBucket = EnvKey { key: "DOCS_BUCKET", decode: pure } + type DatabaseUrl = { prefix :: String, path :: FilePath } -- | The location of the sqlite database. diff --git a/app/src/App/Effect/PackageSets.purs b/app/src/App/Effect/PackageSets.purs index 9309f2487..fa33111af 100644 --- a/app/src/App/Effect/PackageSets.purs +++ b/app/src/App/Effect/PackageSets.purs @@ -24,10 +24,10 @@ import Registry.App.CLI.Purs as Purs import Registry.App.CLI.Tar as Tar import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log +import Registry.App.Effect.PackageStorage (PACKAGE_STORAGE) +import Registry.App.Effect.PackageStorage as PackageStorage import Registry.App.Effect.Registry (REGISTRY_READ) import Registry.App.Effect.Registry as Registry -import Registry.App.Effect.Storage (STORAGE) -import Registry.App.Effect.Storage as Storage import Registry.Foreign.FSExtra as FS.Extra import Registry.ManifestIndex as ManifestIndex import Registry.PackageName as PackageName @@ -86,7 +86,7 @@ type PackageSetsEnv = -- | A handler for the PACKAGE_SETS effect which compiles the package sets and -- | returns the results. -handle :: forall r a. PackageSetsEnv -> PackageSets a -> Run (REGISTRY_READ + STORAGE + LOG + AFF + EFFECT + r) a +handle :: forall r a. PackageSetsEnv -> PackageSets a -> Run (REGISTRY_READ + PACKAGE_STORAGE + LOG + AFF + EFFECT + r) a handle env = case _ of UpgradeAtomic oldSet@(PackageSet { packages }) compiler changes reply -> reply <$> Except.runExcept do Log.info $ "Performing atomic upgrade of package set " <> Version.print (un PackageSet oldSet).version @@ -239,7 +239,7 @@ handle env = case _ of Nothing -> Except.throw $ "No metadata found for " <> formattedName Just { hash, bytes } -> do Log.debug $ "Installing " <> formattedName - Storage.download name version tarballPath { hash, bytes } + PackageStorage.download name version tarballPath { hash, bytes } Tar.extract { cwd: packagesWorkDir, archive: tarballName } FS.Extra.remove tarballPath Run.liftAff $ FS.Aff.rename extractedPath installPath diff --git a/app/src/App/Effect/Storage.purs b/app/src/App/Effect/PackageStorage.purs similarity index 82% rename from app/src/App/Effect/Storage.purs rename to app/src/App/Effect/PackageStorage.purs index ab44f2e92..4700efc36 100644 --- a/app/src/App/Effect/Storage.purs +++ b/app/src/App/Effect/PackageStorage.purs @@ -1,13 +1,13 @@ --- | An effect for reading and writing to the registry storage backend. -module Registry.App.Effect.Storage +-- | An effect for reading and writing package tarballs in the registry storage backend. +module Registry.App.Effect.PackageStorage ( S3Env , IntegrityCheck - , STORAGE - , STORAGE_CACHE - , Storage(..) - , StorageCache - , _storage - , _storageCache + , PACKAGE_STORAGE + , PACKAGE_STORAGE_CACHE + , PackageStorage(..) + , PackageStorageCache + , _packageStorage + , _packageStorageCache , delete , download , handleReadOnly @@ -48,41 +48,41 @@ import Run.Except as Except -- | Expected integrity values for downloaded packages. type IntegrityCheck = { hash :: Sha256, bytes :: Number } --- | The Storage effect, which describes uploading, downloading, and deleting --- | tarballs from the registry storage backend. -data Storage a +-- | The package storage effect, which describes uploading, downloading, and +-- | deleting tarballs from the registry storage backend. +data PackageStorage a = Upload PackageName Version FilePath (Either String Unit -> a) | Download PackageName Version FilePath IntegrityCheck (Either String Unit -> a) | Delete PackageName Version (Either String Unit -> a) | Query PackageName (Either String (Set Version) -> a) -derive instance Functor Storage +derive instance Functor PackageStorage -type STORAGE r = (storage :: Storage | r) +type PACKAGE_STORAGE r = (packageStorage :: PackageStorage | r) -_storage :: Proxy "storage" -_storage = Proxy +_packageStorage :: Proxy "packageStorage" +_packageStorage = Proxy -- | Upload a package tarball to the storage backend from the given path. -upload :: forall r. PackageName -> Version -> FilePath -> Run (STORAGE + EXCEPT String + r) Unit -upload name version file = Except.rethrow =<< Run.lift _storage (Upload name version file identity) +upload :: forall r. PackageName -> Version -> FilePath -> Run (PACKAGE_STORAGE + EXCEPT String + r) Unit +upload name version file = Except.rethrow =<< Run.lift _packageStorage (Upload name version file identity) -- | Download a package tarball from the storage backend to the given path, -- | verifying its integrity against the expected hash and size. -download :: forall r. PackageName -> Version -> FilePath -> IntegrityCheck -> Run (STORAGE + EXCEPT String + r) Unit -download name version file integrity = Except.rethrow =<< Run.lift _storage (Download name version file integrity identity) +download :: forall r. PackageName -> Version -> FilePath -> IntegrityCheck -> Run (PACKAGE_STORAGE + EXCEPT String + r) Unit +download name version file integrity = Except.rethrow =<< Run.lift _packageStorage (Download name version file integrity identity) -- | Delete a package tarball from the storage backend. -delete :: forall r. PackageName -> Version -> Run (STORAGE + EXCEPT String + r) Unit -delete name version = Except.rethrow =<< Run.lift _storage (Delete name version identity) +delete :: forall r. PackageName -> Version -> Run (PACKAGE_STORAGE + EXCEPT String + r) Unit +delete name version = Except.rethrow =<< Run.lift _packageStorage (Delete name version identity) --- | Interpret the STORAGE effect, given a handler. -interpret :: forall r a. (Storage ~> Run r) -> Run (STORAGE + r) a -> Run r a -interpret handler = Run.interpret (Run.on _storage handler Run.send) +-- | Interpret the PACKAGE_STORAGE effect, given a handler. +interpret :: forall r a. (PackageStorage ~> Run r) -> Run (PACKAGE_STORAGE + r) a -> Run r a +interpret handler = Run.interpret (Run.on _packageStorage handler Run.send) -- | Query what tarballs exist for a package in the storage backend -query :: forall r. PackageName -> Run (STORAGE + EXCEPT String + r) (Set Version) -query name = Except.rethrow =<< Run.lift _storage (Query name identity) +query :: forall r. PackageName -> Run (PACKAGE_STORAGE + EXCEPT String + r) (Set Version) +query name = Except.rethrow =<< Run.lift _packageStorage (Query name identity) formatPackagePath :: PackageName -> Version -> String formatPackagePath name version = Array.fold @@ -131,8 +131,8 @@ type S3Env = } -- | Handle package storage using a remote S3 bucket. -handleS3 :: forall r a. S3Env -> Storage a -> Run (RESOURCE_ENV + LOG + AFF + EFFECT + r) a -handleS3 env = Cache.interpret _storageCache (Cache.handleFs env.cache) <<< case _ of +handleS3 :: forall r a. S3Env -> PackageStorage a -> Run (RESOURCE_ENV + LOG + AFF + EFFECT + r) a +handleS3 env = Cache.interpret _packageStorageCache (Cache.handleFs env.cache) <<< case _ of Query name reply -> map (map reply) Except.runExcept do s3 <- connectS3 env.s3 resources <- Except.rethrow =<< Run.liftAff (withRetryListObjects s3 name) @@ -142,10 +142,10 @@ handleS3 env = Cache.interpret _storageCache (Cache.handleFs env.cache) <<< case Download name version path integrity reply -> map (map reply) Except.runExcept do let package = formatPackageVersion name version - buffer <- Cache.get _storageCache (Package name version) >>= case _ of + buffer <- Cache.get _packageStorageCache (Package name version) >>= case _ of Nothing -> do buffer <- downloadS3 name version integrity - Cache.put _storageCache (Package name version) buffer + Cache.put _packageStorageCache (Package name version) buffer pure buffer Just cached -> do archiveHash <- Run.liftEffect $ Sha256.hashBuffer cached @@ -154,9 +154,9 @@ handleS3 env = Cache.interpret _storageCache (Cache.handleFs env.cache) <<< case pure cached else do Log.warn $ "Cached tarball for " <> package <> " failed integrity check, evicting and re-downloading..." - Cache.delete _storageCache (Package name version) + Cache.delete _packageStorageCache (Package name version) buffer <- downloadS3 name version integrity - Cache.put _storageCache (Package name version) buffer + Cache.put _packageStorageCache (Package name version) buffer pure buffer Run.liftAff (Aff.attempt (FS.Aff.writeFile path buffer)) >>= case _ of Left error -> do @@ -216,15 +216,15 @@ handleS3 env = Cache.interpret _storageCache (Cache.handleFs env.cache) <<< case Except.throw $ "Could not delete package " <> package <> " due to an error connecting to the storage backend." Succeeded _ -> do Log.debug $ "Deleted release of " <> package <> " from S3 at the path " <> packagePath - Cache.delete _storageCache (Package name version) + Cache.delete _packageStorageCache (Package name version) pure unit else do Log.error $ packagePath <> " does not exist on S3 (available: " <> String.joinWith ", " published <> ")" Except.throw $ "Could not delete " <> package <> " because it does not exist in the storage backend." -- | A storage effect that reads from the registry but does not write to it. -handleReadOnly :: forall r a. FilePath -> Storage a -> Run (RESOURCE_ENV + LOG + AFF + EFFECT + r) a -handleReadOnly cache = Cache.interpret _storageCache (Cache.handleFs cache) <<< case _ of +handleReadOnly :: forall r a. FilePath -> PackageStorage a -> Run (RESOURCE_ENV + LOG + AFF + EFFECT + r) a +handleReadOnly cache = Cache.interpret _packageStorageCache (Cache.handleFs cache) <<< case _ of -- TODO: is there a way to do this without S3 credentials? Query _ reply -> do pure $ reply $ Left "Cannot query in read-only mode." @@ -239,7 +239,7 @@ handleReadOnly cache = Cache.interpret _storageCache (Cache.handleFs cache) <<< Except.throw $ "Could not cache package " <> package <> " due to a file system error." Right buf -> pure buf - Cache.put _storageCache (Package name version) buffer + Cache.put _packageStorageCache (Package name version) buffer Delete name version reply -> do packageUrl <- formatPackageUrl name version @@ -248,10 +248,10 @@ handleReadOnly cache = Cache.interpret _storageCache (Cache.handleFs cache) <<< Download name version path integrity reply -> map (map reply) Except.runExcept do let package = formatPackageVersion name version - buffer <- Cache.get _storageCache (Package name version) >>= case _ of + buffer <- Cache.get _packageStorageCache (Package name version) >>= case _ of Nothing -> do buffer <- downloadS3 name version integrity - Cache.put _storageCache (Package name version) buffer + Cache.put _packageStorageCache (Package name version) buffer pure buffer Just cached -> do archiveHash <- Run.liftEffect $ Sha256.hashBuffer cached @@ -260,9 +260,9 @@ handleReadOnly cache = Cache.interpret _storageCache (Cache.handleFs cache) <<< pure cached else do Log.warn $ "Cached tarball for " <> package <> " failed integrity check, evicting and re-downloading..." - Cache.delete _storageCache (Package name version) + Cache.delete _packageStorageCache (Package name version) buffer <- downloadS3 name version integrity - Cache.put _storageCache (Package name version) buffer + Cache.put _packageStorageCache (Package name version) buffer pure buffer Run.liftAff (Aff.attempt (FS.Aff.writeFile path buffer)) >>= case _ of Left error -> do @@ -328,19 +328,19 @@ withRetryListObjects s3 name = do Succeeded objects -> pure $ map _.key objects --- | A key type for the storage cache. Only supports packages identified by --- | their name and version. -data StorageCache (c :: Type -> Type -> Type) a = Package PackageName Version (c Buffer a) +-- | A key type for the package storage cache. Only supports packages identified +-- | by their name and version. +data PackageStorageCache (c :: Type -> Type -> Type) a = Package PackageName Version (c Buffer a) -instance Functor2 c => Functor (StorageCache c) where +instance Functor2 c => Functor (PackageStorageCache c) where map k (Package name version a) = Package name version (map2 k a) -instance FsEncodable StorageCache where +instance FsEncodable PackageStorageCache where encodeFs = case _ of Package name version next -> Exists.mkExists $ AsBuffer (PackageName.print name <> "-" <> Version.print version) next -type STORAGE_CACHE r = (storageCache :: Cache StorageCache | r) +type PACKAGE_STORAGE_CACHE r = (packageStorageCache :: Cache PackageStorageCache | r) -_storageCache :: Proxy "storageCache" -_storageCache = Proxy +_packageStorageCache :: Proxy "packageStorageCache" +_packageStorageCache = Proxy diff --git a/app/src/App/Server/Env.purs b/app/src/App/Server/Env.purs index b2310614e..7b8565f1c 100644 --- a/app/src/App/Server/Env.purs +++ b/app/src/App/Server/Env.purs @@ -17,6 +17,8 @@ import Registry.App.Effect.Cache (CacheRef) import Registry.App.Effect.Cache as Cache import Registry.App.Effect.Db (DB) import Registry.App.Effect.Db as Db +import Registry.App.Effect.DocsStorage (DOCS_STORAGE) +import Registry.App.Effect.DocsStorage as DocsStorage import Registry.App.Effect.Env (PACCHETTIBOTTI_ENV, RESOURCE_ENV, ResourceEnv) import Registry.App.Effect.Env as Env import Registry.App.Effect.GitHub (GITHUB) @@ -25,14 +27,14 @@ import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log import Registry.App.Effect.PackageSets (PACKAGE_SETS) import Registry.App.Effect.PackageSets as PackageSets +import Registry.App.Effect.PackageStorage (PACKAGE_STORAGE) +import Registry.App.Effect.PackageStorage as PackageStorage import Registry.App.Effect.Pursuit (PURSUIT) import Registry.App.Effect.Pursuit as Pursuit import Registry.App.Effect.Registry (REGISTRY) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (SOURCE) import Registry.App.Effect.Source as Source -import Registry.App.Effect.Storage (STORAGE) -import Registry.App.Effect.Storage as Storage import Registry.App.SQLite (SQLite) import Registry.App.SQLite as SQLite import Registry.Foreign.FSExtra as FS.Extra @@ -50,6 +52,7 @@ type ServerEnvVars = , privateKey :: String , spacesKey :: String , spacesSecret :: String + , docsBucket :: String , resourceEnv :: ResourceEnv , readOnly :: Boolean } @@ -63,9 +66,10 @@ readServerEnvVars = do privateKey <- Env.lookupRequired Env.pacchettibottiED25519 spacesKey <- Env.lookupRequired Env.spacesKey spacesSecret <- Env.lookupRequired Env.spacesSecret + docsBucket <- Env.lookupWithDefault Env.docsBucket "purescript-registry-docs" resourceEnv <- Env.lookupResourceEnv isReadOnly <- Env.lookupWithDefault Env.readOnly false - pure { token, publicKey, privateKey, spacesKey, spacesSecret, resourceEnv, readOnly: isReadOnly } + pure { token, publicKey, privateKey, spacesKey, spacesSecret, docsBucket, resourceEnv, readOnly: isReadOnly } type ServerEnv = { cacheDir :: FilePath @@ -113,7 +117,7 @@ createServerEnv = do , jobId: Nothing } -type ServerEffects = (RESOURCE_ENV + PACCHETTIBOTTI_ENV + REGISTRY + PACKAGE_SETS + STORAGE + PURSUIT + SOURCE + DB + GITHUB + COMPILER_CACHE + PURS_GRAPH_CACHE + LOG + EXCEPT String + AFF + EFFECT ()) +type ServerEffects = (RESOURCE_ENV + PACCHETTIBOTTI_ENV + REGISTRY + PACKAGE_SETS + PACKAGE_STORAGE + DOCS_STORAGE + PURSUIT + SOURCE + DB + GITHUB + COMPILER_CACHE + PURS_GRAPH_CACHE + LOG + EXCEPT String + AFF + EFFECT ()) runServer :: ServerEnv @@ -156,12 +160,18 @@ runEffects env operation = Aff.attempt do , debouncer: env.debouncer , cacheRef: env.registryCacheRef } + docsStorageEnv = + { s3: { key: env.vars.spacesKey, secret: env.vars.spacesSecret } + , bucket: env.vars.docsBucket + } operation # PackageSets.interpret (PackageSets.handle { workdir: scratchDir }) # Registry.interpretWrite (Registry.handleWrite registryEnv) # Registry.interpretRead (Registry.handleRead registryEnv) # Pursuit.interpret (if env.vars.readOnly then Pursuit.handlePure else Pursuit.handleAff env.vars.token) - # Storage.interpret (if env.vars.readOnly then Storage.handleReadOnly env.cacheDir else Storage.handleS3 { s3: { key: env.vars.spacesKey, secret: env.vars.spacesSecret }, cache: env.cacheDir }) + # PackageStorage.interpret (if env.vars.readOnly then PackageStorage.handleReadOnly env.cacheDir else PackageStorage.handleS3 { s3: { key: env.vars.spacesKey, secret: env.vars.spacesSecret }, cache: env.cacheDir }) + # DocsStorage.interpret + (if env.vars.readOnly then DocsStorage.handleReadOnly docsStorageEnv else DocsStorage.handleS3 docsStorageEnv) # Source.interpret Source.handle # GitHub.interpret (GitHub.handle { octokit: env.octokit, cache: env.cacheDir, ref: env.githubCacheRef }) # Cache.interpret _compilerCache (Cache.handleFs env.cacheDir) diff --git a/app/src/App/Server/MatrixBuilder.purs b/app/src/App/Server/MatrixBuilder.purs index 6e1ebc00c..c4c4b152a 100644 --- a/app/src/App/Server/MatrixBuilder.purs +++ b/app/src/App/Server/MatrixBuilder.purs @@ -30,10 +30,10 @@ import Registry.App.CLI.PursVersions as PursVersions import Registry.App.CLI.Tar as Tar import Registry.App.Effect.Log (LOG) import Registry.App.Effect.Log as Log +import Registry.App.Effect.PackageStorage (PACKAGE_STORAGE) +import Registry.App.Effect.PackageStorage as PackageStorage import Registry.App.Effect.Registry (REGISTRY, REGISTRY_READ) import Registry.App.Effect.Registry as Registry -import Registry.App.Effect.Storage (STORAGE) -import Registry.App.Effect.Storage as Storage import Registry.Foreign.FSExtra as FS.Extra import Registry.Foreign.Tmp as Tmp import Registry.ManifestIndex as ManifestIndex @@ -48,7 +48,7 @@ import Run as Run import Run.Except (EXCEPT) import Run.Except as Except -runMatrixJob :: forall r. MatrixJobData -> Run (REGISTRY + STORAGE + LOG + AFF + EFFECT + EXCEPT String + r) (Map PackageName Range) +runMatrixJob :: forall r. MatrixJobData -> Run (REGISTRY + PACKAGE_STORAGE + LOG + AFF + EFFECT + EXCEPT String + r) (Map PackageName Range) runMatrixJob { compilerVersion, packageName, packageVersion, payload: buildPlan } = do workdir <- Tmp.mkTmpDir let installed = Path.concat [ workdir, ".registry" ] @@ -114,7 +114,7 @@ type BuildPlanEntry = { version :: Version, hash :: Sha256, bytes :: Number } -- | Install all dependencies indicated by the build plan to the specified -- | directory. Packages will be installed at 'dir/package-name-x.y.z'. -installBuildPlan :: forall r. Map PackageName BuildPlanEntry -> FilePath -> Run (STORAGE + LOG + AFF + EXCEPT String + r) Unit +installBuildPlan :: forall r. Map PackageName BuildPlanEntry -> FilePath -> Run (PACKAGE_STORAGE + LOG + AFF + EXCEPT String + r) Unit installBuildPlan resolutions dependenciesDir = do Run.liftAff $ FS.Extra.ensureDirectory dependenciesDir -- We fetch every dependency at its resolved version, unpack the tarball, and @@ -125,7 +125,7 @@ installBuildPlan resolutions dependenciesDir = do -- unpacked, ie. package-name-major.minor.patch filename = PackageName.print name <> "-" <> Version.print version <> ".tar.gz" filepath = Path.concat [ dependenciesDir, filename ] - Storage.download name version filepath { hash, bytes } + PackageStorage.download name version filepath { hash, bytes } Run.liftAff (Aff.attempt (Tar.extract { cwd: dependenciesDir, archive: filename })) >>= case _ of Left error -> do Log.error $ "Failed to unpack " <> filename <> ": " <> Aff.message error diff --git a/app/test/App/API.purs b/app/test/App/API.purs index 0ecd69fd2..8d8cfec97 100644 --- a/app/test/App/API.purs +++ b/app/test/App/API.purs @@ -19,9 +19,9 @@ import Registry.App.API (LicenseValidationError(..), validateLicense) import Registry.App.API as API import Registry.App.Effect.Env as Env import Registry.App.Effect.Log as Log +import Registry.App.Effect.PackageStorage as PackageStorage import Registry.App.Effect.Pursuit as Pursuit import Registry.App.Effect.Registry as Registry -import Registry.App.Effect.Storage as Storage import Registry.App.Legacy.Types (RawPackageName(..)) import Registry.Constants as Constants import Registry.Foreign.FSExtra as FS.Extra @@ -78,9 +78,9 @@ assertPublicationState . PackageName -> Version -> { manifest :: Boolean, metadata :: Boolean, storage :: Boolean } - -> Run (Registry.REGISTRY_READ + Storage.STORAGE + Except.EXCEPT String + r) Unit + -> Run (Registry.REGISTRY_READ + PackageStorage.PACKAGE_STORAGE + Except.EXCEPT String + r) Unit assertPublicationState name version expected = do - storedVersions <- Storage.query name + storedVersions <- PackageStorage.query name maybeMetadata <- Registry.readMetadata name maybeManifest <- Registry.readManifest name version let @@ -107,7 +107,7 @@ runPipelineAssertion env action = do , metadata: env.metadata , pursuitExcludes: Set.empty , username: "jon" - , storage: env.storageDir + , packageStorage: env.storageDir , github: env.githubDir } action @@ -154,7 +154,7 @@ spec = do , metadata , pursuitExcludes: Set.empty , username: "jon" - , storage: storageDir + , packageStorage: storageDir , github: githubDir } @@ -184,7 +184,7 @@ spec = do Left err -> Except.throw $ "Failed to get published versions: " <> err -- As well as to the storage backend - Storage.query name >>= \versions -> + PackageStorage.query name >>= \versions -> unless (Set.member version versions) do Except.throw $ "Expected " <> formatPackageVersion name version <> " to be published to registry storage." @@ -221,7 +221,7 @@ spec = do Right (Right _) -> pure unit Spec.describe "Publication retry reconciliation" do - -- Storage, metadata, and the manifest index are the only durable writes in + -- Package storage, metadata, and the manifest index are the only durable writes in -- the publication pipeline. Failures before storage leave no state to -- reconcile; failures after the manifest happen after core publication is -- complete. These cases cover each partial durable state plus an upload @@ -229,7 +229,7 @@ spec = do Spec.it "Reconciles every partial durable publication state" \env -> do runPipelineAssertion env do Run.liftEffect $ Ref.write - [ Assert.Run.FailStorageUploadAfterWrite + [ Assert.Run.FailPackageStorageUploadAfterWrite , Assert.Run.FailMetadataWrite , Assert.Run.FailManifestWrite ] diff --git a/app/test/App/Effect/DocsStorage.purs b/app/test/App/Effect/DocsStorage.purs new file mode 100644 index 000000000..c512c86e5 --- /dev/null +++ b/app/test/App/Effect/DocsStorage.purs @@ -0,0 +1,109 @@ +module Test.Registry.App.Effect.DocsStorage (spec) where + +import Registry.App.Prelude + +import Data.Codec.JSON as CJ +import Data.Map as Map +import Data.String as String +import Effect.Aff as Aff +import Node.FS.Sync as FS.Sync +import Node.Path as Path +import Registry.App.Effect.DocsStorage (DOCS_STORAGE) +import Registry.App.Effect.DocsStorage as DocsStorage +import Registry.Docgen.Codec as Docgen.Codec +import Registry.Docgen.Docs (DocPackage(..), SourceArtifact(..), schemaVersion) +import Registry.Foreign.FSExtra as FS.Extra +import Registry.Foreign.Tmp as Tmp +import Registry.License as License +import Registry.PackageName (PackageName) +import Registry.Test.Assert as Assert +import Registry.Test.Fixtures (defaultHash, defaultLocation) +import Registry.Test.Utils as Utils +import Registry.Version (Version) +import Run (AFF, EFFECT, Run) +import Run as Run +import Run.Except (EXCEPT) +import Run.Except as Except +import Test.Spec as Spec + +spec :: Spec.Spec Unit +spec = do + Spec.it "supports the complete replaceable documentation lifecycle" do + Aff.bracket Tmp.mkTmpDir FS.Extra.remove \tmp -> do + let name = Utils.unsafePackageName "example" + let version = Utils.unsafeVersion "1.0.0" + let path = Path.concat [ tmp, DocsStorage.formatDocsPath name version ] + + runFs tmp (DocsStorage.exists name version) >>= (_ `Assert.shouldEqual` false) + runFs tmp $ DocsStorage.upload docs + runFs tmp (DocsStorage.exists name version) >>= (_ `Assert.shouldEqual` true) + liftEffect (FS.Sync.exists path) >>= (_ `Assert.shouldEqual` true) + + stored <- runFs tmp $ DocsStorage.download name version + CJ.encode Docgen.Codec.docPackage stored `Assert.shouldEqual` CJ.encode Docgen.Codec.docPackage docs + + duplicate <- Aff.attempt $ runFs tmp $ DocsStorage.upload docs + case duplicate of + Left error -> String.contains (String.Pattern "already exists") (Aff.message error) `Assert.shouldEqual` true + Right _ -> Assert.fail "Immutable upload unexpectedly replaced documentation" + + writeJsonFile Docgen.Codec.docPackage path mismatched + mismatch <- Aff.attempt $ runFs tmp $ DocsStorage.download name version + case mismatch of + Left error -> String.contains (String.Pattern "identifies itself as other@1.0.0") (Aff.message error) `Assert.shouldEqual` true + Right _ -> Assert.fail "Download unexpectedly accepted mismatched artifact identity" + + runFs tmp $ DocsStorage.replace replacement + replaced <- runFs tmp $ DocsStorage.download name version + case replaced of + DocPackage { description: Just "Replacement" } -> pure unit + _ -> Assert.fail "Replacement documentation was not stored" + + runFs tmp $ DocsStorage.delete name version + runFs tmp (DocsStorage.exists name version) >>= (_ `Assert.shouldEqual` false) + runFs tmp $ DocsStorage.delete name version + +runFs + :: forall a + . FilePath + -> Run (DOCS_STORAGE + EXCEPT String + AFF + EFFECT + ()) a + -> Aff a +runFs root = + DocsStorage.interpret (DocsStorage.handleFs root) + >>> Except.catch (\error -> Run.liftAff $ Aff.throwError $ Aff.error error) + >>> Run.runBaseAff' + +docs :: DocPackage +docs = DocPackage + { schemaVersion + , compilerVersion: compilerVersion + , sourceArtifact: SourceArtifact { bytes: 42.0, hash: defaultHash } + , dependencies: Map.empty + , description: Nothing + , license: Utils.fromRight "license" $ License.parse "BSD-3-Clause" + , location: defaultLocation + , locationRef: Nothing + , modules: [] + , name: packageName + , readme: Nothing + , resolvedDependencies: Map.empty + , resolvedModulePackages: Map.empty + , version: packageVersion + } + +replacement :: DocPackage +replacement = case docs of + DocPackage package -> DocPackage package { description = Just "Replacement" } + +mismatched :: DocPackage +mismatched = case docs of + DocPackage package -> DocPackage package { name = Utils.unsafePackageName "other" } + +packageName :: PackageName +packageName = Utils.unsafePackageName "example" + +packageVersion :: Version +packageVersion = Utils.unsafeVersion "1.0.0" + +compilerVersion :: Version +compilerVersion = Utils.unsafeVersion "0.15.15" diff --git a/app/test/Main.purs b/app/test/Main.purs index a7b0e622c..4d0d541a1 100644 --- a/app/test/Main.purs +++ b/app/test/Main.purs @@ -9,6 +9,7 @@ import Test.Registry.App.CLI.Licensee as Test.CLI.Licensee import Test.Registry.App.CLI.Purs as Test.CLI.Purs import Test.Registry.App.CLI.PursVersions as Test.CLI.PursVersions import Test.Registry.App.CLI.Tar as Test.CLI.Tar +import Test.Registry.App.Effect.DocsStorage as Test.Effect.DocsStorage import Test.Registry.App.Effect.PackageSets as Test.Effect.PackageSets import Test.Registry.App.Effect.Registry as Test.Effect.Registry import Test.Registry.App.GitHubIssue as Test.GitHubIssue @@ -38,6 +39,7 @@ main = runSpecAndExitProcess' config [ consoleReporter ] do Spec.describe "PursVersions" Test.CLI.PursVersions.spec Spec.describe "Registry.App.Effect" do + Spec.describe "DocsStorage" Test.Effect.DocsStorage.spec Test.Effect.PackageSets.spec Spec.describe "Registry" Test.Effect.Registry.spec diff --git a/app/test/Test/Assert/Run.purs b/app/test/Test/Assert/Run.purs index a427bd841..328972241 100644 --- a/app/test/Test/Assert/Run.purs +++ b/app/test/Test/Assert/Run.purs @@ -41,14 +41,14 @@ import Registry.App.Effect.Log (LOG, Log(..)) import Registry.App.Effect.Log as Log import Registry.App.Effect.PackageSets (PACKAGE_SETS, PackageSets(..)) import Registry.App.Effect.PackageSets as PackageSets +import Registry.App.Effect.PackageStorage (PACKAGE_STORAGE, PackageStorage) +import Registry.App.Effect.PackageStorage as PackageStorage import Registry.App.Effect.Pursuit (PURSUIT, Pursuit(..)) import Registry.App.Effect.Pursuit as Pursuit import Registry.App.Effect.Registry (REGISTRY, RegistryRead(..), RegistryWrite(..)) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (FetchError(..), SOURCE, Source(..)) import Registry.App.Effect.Source as Source -import Registry.App.Effect.Storage (STORAGE, Storage) -import Registry.App.Effect.Storage as Storage import Registry.App.Prelude as Either import Registry.Foreign.FSExtra as FS.Extra import Registry.Foreign.Octokit (GitHubError(..), IssueNumber(..)) @@ -80,7 +80,7 @@ type TEST_EFFECTS = ( PURSUIT + REGISTRY + PACKAGE_SETS - + STORAGE + + PACKAGE_STORAGE + SOURCE + GITHUB + PACCHETTIBOTTI_ENV @@ -100,7 +100,7 @@ type TEST_EFFECTS = -- | from the front of a plan, allowing one test to fail successive publication -- | retries at each durable boundary while preserving their shared state. data TestFailure - = FailStorageUploadAfterWrite + = FailPackageStorageUploadAfterWrite | FailMetadataWrite | FailManifestWrite @@ -123,7 +123,7 @@ type TestEnv = , metadata :: Ref (Map PackageName Metadata) , index :: Ref ManifestIndex , pursuitExcludes :: Set PackageName - , storage :: FilePath + , packageStorage :: FilePath , github :: FilePath , username :: String } @@ -149,7 +149,7 @@ runTestEffects env operation = Aff.attempt do } ) # PackageSets.interpret handlePackageSetsMock - # Storage.interpret (handleStorageMock { storage: env.storage, failurePlan: env.failurePlan }) + # PackageStorage.interpret (handlePackageStorageMock { packageStorage: env.packageStorage, failurePlan: env.failurePlan }) # Source.interpret (handleSourceMock { github: env.github }) # GitHub.interpret (handleGitHubMock { github: env.github }) -- Environments @@ -291,21 +291,21 @@ handlePackageSetsMock = case _ of UpgradeSequential packageSet _compilerVersion changeSet reply -> pure $ reply $ Right $ Just { failed: changeSet, succeeded: changeSet, result: packageSet } -type StorageMockEnv = - { storage :: FilePath +type PackageStorageMockEnv = + { packageStorage :: FilePath , failurePlan :: Ref (Array TestFailure) } -- We handle the storage effect by copying files to/from the provided -- upload/download directories, and listing versions based on the filenames. -handleStorageMock :: forall r a. StorageMockEnv -> Storage a -> Run (AFF + EFFECT + r) a -handleStorageMock env = case _ of - Storage.Upload name version sourcePath reply -> do - let destinationPath = Path.concat [ env.storage, PackageName.print name <> "-" <> Version.print version <> ".tar.gz" ] +handlePackageStorageMock :: forall r a. PackageStorageMockEnv -> PackageStorage a -> Run (AFF + EFFECT + r) a +handlePackageStorageMock env = case _ of + PackageStorage.Upload name version sourcePath reply -> do + let destinationPath = Path.concat [ env.packageStorage, PackageName.print name <> "-" <> Version.print version <> ".tar.gz" ] Run.liftAff (Aff.attempt (FS.Aff.stat destinationPath)) >>= case _ of Left _ -> do Run.liftAff $ FS.Extra.copy { from: sourcePath, to: destinationPath, preserveTimestamps: true } - failUpload <- consumeFailure FailStorageUploadAfterWrite env.failurePlan + failUpload <- consumeFailure FailPackageStorageUploadAfterWrite env.failurePlan if failUpload then pure $ reply $ Left "Injected storage upload failure after writing the tarball." else @@ -313,10 +313,10 @@ handleStorageMock env = case _ of Right _ -> pure $ reply $ Left $ "Cannot upload " <> formatPackageVersion name version <> " because it already exists in storage at path " <> destinationPath - Storage.Download name version destinationPath integrity reply -> do + PackageStorage.Download name version destinationPath integrity reply -> do let package = formatPackageVersion name version - sourcePath = Path.concat [ env.storage, PackageName.print name <> "-" <> Version.print version <> ".tar.gz" ] + sourcePath = Path.concat [ env.packageStorage, PackageName.print name <> "-" <> Version.print version <> ".tar.gz" ] Run.liftAff (Aff.attempt (FS.Aff.readFile sourcePath)) >>= case _ of Left _ -> pure $ reply $ Left $ "Cannot copy " <> sourcePath <> " because it does not exist in download directory." Right buffer -> do @@ -334,16 +334,16 @@ handleStorageMock env = case _ of Run.liftAff $ FS.Aff.writeFile destinationPath buffer pure $ reply $ Right unit - Storage.Delete name version reply -> do - let sourcePath = Path.concat [ env.storage, PackageName.print name <> "-" <> Version.print version <> ".tar.gz" ] + PackageStorage.Delete name version reply -> do + let sourcePath = Path.concat [ env.packageStorage, PackageName.print name <> "-" <> Version.print version <> ".tar.gz" ] Run.liftAff (Aff.attempt (FS.Aff.stat sourcePath)) >>= case _ of Left _ -> pure $ reply $ Left $ "Cannot delete " <> sourcePath <> " because it does not exist in download directory." Right _ -> do Run.liftAff $ FS.Extra.remove sourcePath pure $ reply $ Right unit - Storage.Query name reply -> do - paths <- Run.liftAff $ FS.Aff.readdir env.storage + PackageStorage.Query name reply -> do + paths <- Run.liftAff $ FS.Aff.readdir env.packageStorage let extractVersion = String.stripPrefix (String.Pattern (PackageName.print name <> "-")) diff --git a/docgen/AGENTS.md b/docgen/AGENTS.md new file mode 100644 index 000000000..48b4c8945 --- /dev/null +++ b/docgen/AGENTS.md @@ -0,0 +1,25 @@ +# Docgen guidance + +- Treat the canonical package codec as a persisted compatibility boundary. + Stored-format changes require an explicit schema-version decision and codec + tests for both accepted and rejected versions. +- Keep `registry-docgen` deterministic from explicit inputs. Filesystem and + network acquisition, compiler invocation, object storage, and runnable tools + belong in `app` or `scripts`, not this package. +- Source spans must be relative to the package tarball root. Do not assume a + package uses a `src` directory or that its source is in a Git checkout. +- Keep Pursuit HTTP behavior and search indexing out of this package. They are + consumers of the canonical artifact. +- Test canonical codec round trips, representative historical JSON conversion, + and direct, selective, aliased, transitive, missing, and cyclic re-exports. + Re-export failures must be reported; documentation modules must never be + silently omitted. +- Schema version 1 records package and compiler versions, source archive + identity, declared and exact dependencies, module ownership, resolved + re-exports, README content and format, source location provenance, and + package-relative source spans. Dependency ranges are retained only for + historical display; use registry manifests when solving ranges. +- Canonical documentation is stored separately from immutable package tarballs + through `Registry.App.Effect.DocsStorage`, at `/.json` in + `DOCS_BUCKET`. Initial uploads are create-only; replacement must remain an + explicit operation because documentation is derived data. diff --git a/docgen/README.md b/docgen/README.md new file mode 100644 index 000000000..1f7c05791 --- /dev/null +++ b/docgen/README.md @@ -0,0 +1,65 @@ +# Registry documentation + +The `registry-docgen` package contains the shared documentation infrastructure +for the PureScript Registry and Pursuit. It owns the compiler-independent +documentation model used by the registry, conversion from historical compiler +and Pursuit JSON, re-export resolution, and rendering package documentation to +HTML. + +The package began as +[`purescript-registry-docgen`](https://github.com/natefaubion/purescript-registry-docgen) +by Nathan Faubion. It is maintained here so documentation generation, storage, +search, and the Pursuit application can evolve together with the registry. + +## Architecture + +The intended data flow is: + +```text +compiler docs JSON + package sources + registry metadata + | + v + canonical package documentation JSON + | + +--------------+---------------+ + | | | + v v v + Pursuit search index HTML renderer +``` + +The important ownership boundary is that the registry owns the canonical +artifact. Pursuit, search, and static rendering consume it; they do not define +it. This is why `docgen` is a top-level workspace package rather than part of a +Pursuit application or the public `registry-lib` package. + +The main modules are: + +- `Registry.Docgen.Docs`: the canonical in-memory documentation model. +- `Registry.Docgen.Codec`: codecs for the canonical JSON artifact. +- `Registry.Docgen.Legacy.*`: codecs for historical compiler and Pursuit JSON. +- `Registry.Docgen.Convert`: conversion from historical JSON to the canonical + model. +- `Registry.Docgen.Generate`: deterministic validation and assembly of a + canonical package artifact from compiler docs, parsed source headers, + package-relative paths, and registry metadata. +- `Registry.Docgen.Reexports`: resolution of module re-exports using package + source headers. +- `Registry.Docgen.Package.Render`: package and module HTML rendering. + +Filesystem access, compiler invocation, source archive handling, README +acquisition, object storage, and executable migration tools belong in `app` or +`scripts`. The `docgen` package should remain deterministic for the same +explicit inputs and should not depend on a Git checkout or a forge API. + +## Development + +From the repository root, in the Nix development environment: + +```sh +spago build -p registry-docgen +spago test -p registry-docgen +purs-tidy check docgen +``` + +Outside an interactive shell which has loaded `direnv`, prefix these commands +with `direnv exec .`. diff --git a/docgen/fixtures/undefined-1.0.2.json b/docgen/fixtures/undefined-1.0.2.json new file mode 100644 index 000000000..cd4ab33a4 --- /dev/null +++ b/docgen/fixtures/undefined-1.0.2.json @@ -0,0 +1 @@ +{"compilerVersion":"0.12.2","dependencies":{},"description":"Package containing the undefined value.","license":"MIT","location":{"githubOwner":"bklaric","githubRepo":"purescript-undefined"},"locationRef":"v1.0.2","modules":[{"declarations":[{"children":[],"info":{"tag":"value","value":{"name":{"moduleName":"Undefined","name":"undefined"},"signature":{"tag":"forall","value":{"bindings":[{"isVisible":false,"name":"anything"}],"body":{"tag":"ident","value":"anything"}}}}},"sourceSpan":{"end":{"column":46,"line":3},"path":"custom/Undefined.purs","start":{"column":1,"line":3}}}],"name":"Undefined","reexports":[]}],"name":"undefined","readme":{"content":"# Undefined","extension":"md"},"resolvedDependencies":{},"resolvedModulePackages":{},"schemaVersion":1,"sourceArtifact":{"bytes":42,"hash":"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="},"version":"1.0.2"} diff --git a/docgen/package.json b/docgen/package.json new file mode 100644 index 000000000..b7902a6e2 --- /dev/null +++ b/docgen/package.json @@ -0,0 +1,9 @@ +{ + "name": "registry-docgen", + "private": true, + "type": "module", + "license": "BSD-3-Clause", + "dependencies": { + "commonmark": "^0.31.2" + } +} diff --git a/docgen/spago.yaml b/docgen/spago.yaml new file mode 100644 index 000000000..fad368f12 --- /dev/null +++ b/docgen/spago.yaml @@ -0,0 +1,39 @@ +package: + name: registry-docgen + publish: + license: BSD-3-Clause + version: 0.0.1 + dependencies: + - arrays + - codec + - codec-json + - control + - either + - filterable + - foldable-traversable + - functions + - json + - language-cst-parser + - lazy + - lists + - maybe + - newtype + - ordered-collections + - prelude + - profunctor + - profunctor-lenses + - registry-lib + - safe-coerce + - strings + - transformers + - tuples + - versions + test: + main: Test.Registry.Docgen + dependencies: + - aff + - node-buffer + - node-fs + - registry-test-utils + - spec + - spec-node diff --git a/docgen/src/Registry/Docgen/Codec.purs b/docgen/src/Registry/Docgen/Codec.purs new file mode 100644 index 000000000..72ffc28fd --- /dev/null +++ b/docgen/src/Registry/Docgen/Codec.purs @@ -0,0 +1,505 @@ +module Registry.Docgen.Codec where + +import Prelude + +import Codec.JSON.DecodeError as DecodeError +import Control.Alt ((<|>)) +import Data.Codec (Codec') +import Data.Codec as Codec +import Data.Codec.JSON (Codec) +import Data.Codec.JSON.Common as Common +import Data.Codec.JSON.Record (object, optional) +import Data.Either (Either(..)) +import Data.Lazy as Lazy +import Data.Lens (traversed) +import Data.Lens.Iso.Newtype (_Newtype) +import Data.Newtype (unwrap) +import Data.Tuple (Tuple(..)) +import JSON (JSON) +import JSON as JSON +import JSON.Object as JObject +import Registry.Docgen.Decoder (Decoder, decodeArray, decodeInt, decodeJObject, decodeNull, decodeProp, decodeString) +import Registry.Docgen.Decoder as Decoder +import Registry.Docgen.Docs (Associativity(..), DataConstructorName, DocChildDeclaration, DocChildDeclarationInfo(..), DocConstraint, DocDeclaration, DocDeclarationInfo(..), DocModule, DocPackage(..), DocReexport, DocType(..), ForallBinding, FunDep, Ident, InfixAlias(..), IntLiteral(..), ModuleName(..), OperatorName, Qualified, RawRange, Readme, Role(..), RowLabel, SourceArtifact, SourcePos, SourceSpan, StringLiteral(..), TypeName, TypeVar, ValueName, schemaVersion) +import Registry.Internal.Codec as Internal.Codec +import Registry.License as License +import Registry.Location as Location +import Registry.PackageName as PackageName +import Registry.Sha256 as Sha256 +import Registry.Version as Version + +defer :: forall m a b. (Unit -> Codec' m a b) -> Codec' m a b +defer k = + Codec.codec' + (\a -> Codec.decode (Lazy.force lazy) a) + (\a -> Codec.encode (Lazy.force lazy) a) + where + lazy = Lazy.defer k + +taggedSum :: forall a. (String -> Decoder JSON a) -> (a -> Tuple String JSON) -> Codec a +taggedSum decode' encode' = Codec.codec' decode encode + where + decode = unwrap $ decodeJObject do + tag <- decodeProp "tag" decodeString + decodeProp "value" (decode' tag) + + encode a = do + let Tuple tag value = encode' a + JSON.fromJObject $ JObject.fromEntries + [ Tuple "tag" (JSON.fromString tag) + , Tuple "value" value + ] + +moduleName :: Codec ModuleName +moduleName = _Newtype Common.string + +rawRange :: Codec RawRange +rawRange = _Newtype Common.string + +ident :: Codec Ident +ident = _Newtype Common.string + +typeName :: Codec TypeName +typeName = _Newtype Common.string + +operatorName :: Codec OperatorName +operatorName = _Newtype Common.string + +dataConstructorName :: Codec DataConstructorName +dataConstructorName = _Newtype Common.string + +valueName :: Codec ValueName +valueName = _Newtype Common.string + +readme :: Codec Readme +readme = _Newtype $ object + { content: Common.string + , extension: optional Common.string + } + +sourcePos :: Codec SourcePos +sourcePos = _Newtype $ object + { column: Common.int + , line: Common.int + } + +sourceSpan :: Codec SourceSpan +sourceSpan = _Newtype $ object + { end: sourcePos + , path: Common.string + , start: sourcePos + } + +qualified :: forall a. Codec a -> Codec (Qualified a) +qualified name = _Newtype $ object { moduleName, name } + +typeVar :: Codec TypeVar +typeVar = _Newtype $ object + { ident + , signature: optional docType + } + +funDep :: Codec FunDep +funDep = _Newtype $ object + { determiners: Common.array ident + , determinees: Common.array ident + } + +stringLiteral :: Codec StringLiteral +stringLiteral = Decoder.toCodec decode encode + where + decode = + StringValue <$> decodeString + <|> StringCodeUnits <$> decodeArray (traversed decodeInt) + + encode = case _ of + StringValue value -> + JSON.fromString value + StringCodeUnits value -> + JSON.fromArray $ JSON.fromInt <$> value + +intLiteral :: Codec IntLiteral +intLiteral = Decoder.toCodec decode encode + where + decode = + IntSmall <$> decodeInt + <|> IntBig <$> decodeString + + encode = case _ of + IntSmall value -> + JSON.fromInt value + IntBig value -> + JSON.fromString value + +forallBinding :: Codec ForallBinding +forallBinding = _Newtype $ defer \_ -> object + { isVisible: Common.boolean + , name: ident + , signature: optional docType + } + +rowLabel :: Codec RowLabel +rowLabel = _Newtype $ defer \_ -> object + { label: stringLiteral + , signature: docType + } + +role :: Codec Role +role = Decoder.toCodec decode encode + where + decode = do + tag <- decodeString + case tag of + "nominal" -> pure Nominal + "representational" -> pure Representational + "phantom" -> pure Phantom + other -> Decoder.throw $ DecodeError.basic $ "Unknown Role: " <> other + + encode = case _ of + Nominal -> JSON.fromString "nominal" + Representational -> JSON.fromString "representational" + Phantom -> JSON.fromString "phantom" + +docConstraint :: Codec DocConstraint +docConstraint = _Newtype $ defer \_ -> object + { args: Common.array docType + , name: qualified typeName + } + +docType :: Codec DocType +docType = defer \_ -> taggedSum decode encode + where + decode = case _ of + "wildcard" -> + TypeWildcard <$ decodeNull + "ident" -> + TypeIdent <$> Decoder.fromCodec ident + "string" -> + TypeString <$> Decoder.fromCodec stringLiteral + "int" -> + TypeInt <$> Decoder.fromCodec intLiteral + "constructor" -> + TypeConstructor <$> Decoder.fromCodec typeConstructorRep + "operator" -> + TypeOperator <$> Decoder.fromCodec typeOperatorRep + "app" -> + TypeApp <$> Decoder.fromCodec typeAppRep + "kindApp" -> + TypeKindApp <$> Decoder.fromCodec typeAppRep + "infixApp" -> + TypeInfixApp <$> Decoder.fromCodec typeInfixAppRep + "forall" -> + TypeForall <$> Decoder.fromCodec typeForallRep + "constrained" -> + TypeConstrained <$> Decoder.fromCodec typeConstrainedRep + "function" -> + TypeFunction <$> Decoder.fromCodec typeFunctionRep + "kindSignature" -> + TypeKindSignature <$> Decoder.fromCodec typeKindSignatureRep + "record" -> + TypeRecord <$> Decoder.fromCodec typeRowRep + "row" -> + TypeRow <$> Decoder.fromCodec typeRowRep + "parens" -> + TypeParens <$> Decoder.fromCodec docType + other -> + Decoder.throw $ DecodeError.basic $ "Unknown DocType: " <> other + + encode = case _ of + TypeWildcard -> + Tuple "wildcard" JSON.null + TypeIdent value -> + Tuple "ident" (Codec.encode ident value) + TypeString value -> + Tuple "string" (Codec.encode stringLiteral value) + TypeInt value -> + Tuple "int" (Codec.encode intLiteral value) + TypeConstructor value -> + Tuple "constructor" (Codec.encode typeConstructorRep value) + TypeOperator value -> + Tuple "operator" (Codec.encode typeOperatorRep value) + TypeApp value -> + Tuple "app" (Codec.encode typeAppRep value) + TypeKindApp value -> + Tuple "kindApp" (Codec.encode typeAppRep value) + TypeInfixApp value -> + Tuple "infixApp" (Codec.encode typeInfixAppRep value) + TypeForall value -> + Tuple "forall" (Codec.encode typeForallRep value) + TypeConstrained value -> + Tuple "constrained" (Codec.encode typeConstrainedRep value) + TypeFunction value -> + Tuple "function" (Codec.encode typeFunctionRep value) + TypeKindSignature value -> + Tuple "kindSignature" (Codec.encode typeKindSignatureRep value) + TypeRecord value -> + Tuple "record" (Codec.encode typeRowRep value) + TypeRow value -> + Tuple "row" (Codec.encode typeRowRep value) + TypeParens value -> + Tuple "parens" (Codec.encode docType value) + + typeConstructorRep = + qualified typeName + + typeOperatorRep = + qualified operatorName + + typeAppRep = defer \_ -> object + { arg: docType + , function: docType + } + + typeInfixAppRep = defer \_ -> object + { argLhs: docType + , argRhs: docType + , operator: docType + } + + typeForallRep = defer \_ -> object + { bindings: Common.nonEmptyArray forallBinding + , body: docType + } + + typeConstrainedRep = defer \_ -> object + { constraint: docConstraint + , result: docType + } + + typeFunctionRep = defer \_ -> object + { arg: docType + , result: docType + } + + typeKindSignatureRep = defer \_ -> object + { signature: docType + , term: docType + } + + typeRowRep = defer \_ -> object + { labels: Common.array rowLabel + , tail: optional docType + } + +docChildDeclarationInfo :: Codec DocChildDeclarationInfo +docChildDeclarationInfo = taggedSum decode encode + where + decode = case _ of + "instance" -> + ChildDeclInstance <$> Decoder.fromCodec childDeclInstanceRep + "constructor" -> + ChildDeclConstructor <$> Decoder.fromCodec childDeclConstructorRep + "typeClassMember" -> + ChildDeclTypeClassMember <$> Decoder.fromCodec childDeclTypeClassMemberRep + other -> + Decoder.throw $ DecodeError.basic $ "Unknown ChildDeclarationInfo: " <> other + + encode = case _ of + ChildDeclInstance value -> + Tuple "instance" $ Codec.encode childDeclInstanceRep value + ChildDeclConstructor value -> + Tuple "constructor" $ Codec.encode childDeclConstructorRep value + ChildDeclTypeClassMember value -> + Tuple "typeClassMember" $ Codec.encode childDeclTypeClassMemberRep value + + childDeclInstanceRep = object + { constraints: Common.array docConstraint + , head: docType + , name: qualified valueName + } + + childDeclConstructorRep = object + { name: qualified dataConstructorName + , args: Common.array docType + } + + childDeclTypeClassMemberRep = object + { name: qualified valueName + , signature: docType + } + +docChildDeclaration :: Codec DocChildDeclaration +docChildDeclaration = _Newtype $ object + { comments: optional Common.string + , info: docChildDeclarationInfo + , sourceSpan: optional sourceSpan + } + +infixAlias :: Codec InfixAlias +infixAlias = taggedSum decode encode + where + decode = case _ of + "type" -> + AliasType <$> Decoder.fromCodec aliasTypeRep + "constructor" -> + AliasConstructor <$> Decoder.fromCodec aliasConstructorRep + "value" -> + AliasValue <$> Decoder.fromCodec aliasValueRep + other -> + Decoder.throw $ DecodeError.basic $ "Unknown InfixAlias: " <> other + + encode = case _ of + AliasType value -> + Tuple "type" $ Codec.encode aliasTypeRep value + AliasConstructor value -> + Tuple "constructor" $ Codec.encode aliasConstructorRep value + AliasValue value -> + Tuple "value" $ Codec.encode aliasValueRep value + + aliasTypeRep = + qualified typeName + + aliasConstructorRep = + qualified dataConstructorName + + aliasValueRep = + qualified valueName + +associativity :: Codec Associativity +associativity = Decoder.toCodec decode encode + where + decode = do + tag <- decodeString + case tag of + "infixl" -> pure Infixl + "infixr" -> pure Infixr + "infix" -> pure Infix + other -> Decoder.throw $ DecodeError.basic $ "Unknown Associativity: " <> other + + encode = case _ of + Infixl -> JSON.fromString "infixl" + Infixr -> JSON.fromString "infixr" + Infix -> JSON.fromString "infix" + +docDeclarationInfo :: Codec DocDeclarationInfo +docDeclarationInfo = taggedSum decode encode + where + decode = case _ of + "value" -> + DeclValue <$> Decoder.fromCodec declValueRep + "data" -> + DeclData <$> Decoder.fromCodec declDataRep + "type" -> + DeclType <$> Decoder.fromCodec declTypeRep + "typeClass" -> + DeclTypeClass <$> Decoder.fromCodec declTypeClassRep + "infix" -> + DeclInfix <$> Decoder.fromCodec declInfixRep + "foreignData" -> + DeclForeignData <$> Decoder.fromCodec declForeignDataRep + other -> + Decoder.throw $ DecodeError.basic $ "Unknown DeclarationInfo: " <> other + + encode = case _ of + DeclValue value -> + Tuple "value" $ Codec.encode declValueRep value + DeclData value -> + Tuple "data" $ Codec.encode declDataRep value + DeclType value -> + Tuple "type" $ Codec.encode declTypeRep value + DeclTypeClass value -> + Tuple "typeClass" $ Codec.encode declTypeClassRep value + DeclInfix value -> + Tuple "infix" $ Codec.encode declInfixRep value + DeclForeignData value -> + Tuple "foreignData" $ Codec.encode declForeignDataRep value + + declValueRep = object + { name: qualified valueName + , signature: docType + } + + declDataRep = object + { isNewtype: Common.boolean + , name: qualified typeName + , roles: Common.array role + , signature: optional docType + , vars: Common.array typeVar + } + + declTypeRep = object + { body: docType + , name: qualified typeName + , signature: optional docType + , vars: Common.array typeVar + } + + declTypeClassRep = object + { funDeps: Common.array funDep + , name: qualified typeName + , signature: optional docType + , superClasses: Common.array docConstraint + , vars: Common.array typeVar + } + + declInfixRep = object + { alias: infixAlias + , associativity + , name: qualified operatorName + , precedence: Common.int + } + + declForeignDataRep = object + { name: qualified typeName + , roles: Common.array role + , signature: docType + } + +docDeclaration :: Codec DocDeclaration +docDeclaration = _Newtype $ object + { children: Common.array docChildDeclaration + , info: docDeclarationInfo + , comments: optional Common.string + , sourceSpan: optional sourceSpan + } + +docReexport :: Codec DocReexport +docReexport = _Newtype $ object + { moduleName + , declarations: Common.array docDeclaration + } + +docModule :: Codec DocModule +docModule = _Newtype $ object + { comments: optional Common.string + , declarations: Common.array docDeclaration + , name: moduleName + , reexports: Common.array docReexport + } + +sourceArtifact :: Codec SourceArtifact +sourceArtifact = _Newtype $ object + { bytes: Common.number + , hash: Sha256.codec + } + +docPackage :: Codec DocPackage +docPackage = Decoder.toCodec decode encode + where + rep = object + { schemaVersion: Common.int + , compilerVersion: Version.codec + , sourceArtifact + , dependencies: Internal.Codec.packageMap rawRange + , description: optional Common.string + , license: License.codec + , location: Location.codec + , locationRef: optional Common.string + , modules: Common.array docModule + , name: PackageName.codec + , readme: optional readme + , resolvedDependencies: Internal.Codec.packageMap Version.codec + , resolvedModulePackages: Internal.Codec.strMap "ModuleName" (Right <<< ModuleName) (\(ModuleName name) -> name) PackageName.codec + , version: Version.codec + } + + decode = do + { schemaVersion: version } <- Decoder.fromCodec $ object { schemaVersion: Common.int } + if version /= schemaVersion then + Decoder.throw $ DecodeError.basic $ + "Unsupported documentation schema version " <> show version + else do + value <- Decoder.fromCodec rep + pure $ DocPackage value + + encode (DocPackage value) = Codec.encode rep value diff --git a/docgen/src/Registry/Docgen/Commonmark.js b/docgen/src/Registry/Docgen/Commonmark.js new file mode 100644 index 000000000..552261246 --- /dev/null +++ b/docgen/src/Registry/Docgen/Commonmark.js @@ -0,0 +1,46 @@ +import * as commonmark from 'commonmark'; + +const absoluteUriRegex = /^(https|mailto):/i; + +function isAllowedUri(uri) { + // return uri.startsWith('#') || uri.startsWith('/') || absoluteUriRegex.test(uri); + return absoluteUriRegex.test(uri); +} + +function sanitize(md) { + const walker = md.walker(); + + while (true) { + const event = walker.next(); + + if (!event) { + break; + } + + const node = event.node; + + if ( + !event.entering && + node.type === 'link' && + node.destination && + !isAllowedUri(node.destination) + ) { + while (node.firstChild) { + node.insertBefore(node.firstChild); + } + node.unlink(); + } + } + + return md; +} + +export function _renderMarkdownHTML(options, str) { + const md = new commonmark.Parser().parse(str); + + if (options.safe) { + sanitize(md); + } + + return new commonmark.HtmlRenderer({ safe: options.safe, smart: true }).render(md); +} diff --git a/docgen/src/Registry/Docgen/Commonmark.purs b/docgen/src/Registry/Docgen/Commonmark.purs new file mode 100644 index 000000000..ff849c548 --- /dev/null +++ b/docgen/src/Registry/Docgen/Commonmark.purs @@ -0,0 +1,14 @@ +module Registry.Docgen.Commonmark + ( MarkdownOptions + , renderMarkdownHTML + ) where + +import Data.Function.Uncurried (Fn2, runFn2) +import Registry.Docgen.HTML (HTML) + +type MarkdownOptions = { safe :: Boolean } + +foreign import _renderMarkdownHTML :: Fn2 MarkdownOptions String HTML + +renderMarkdownHTML :: MarkdownOptions -> String -> HTML +renderMarkdownHTML = runFn2 _renderMarkdownHTML diff --git a/docgen/src/Registry/Docgen/Convert.purs b/docgen/src/Registry/Docgen/Convert.purs new file mode 100644 index 000000000..e0846c0d1 --- /dev/null +++ b/docgen/src/Registry/Docgen/Convert.purs @@ -0,0 +1,337 @@ +module Registry.Docgen.Convert + ( fromLegacyPackage + , fromLegacyModule + ) where + +import Prelude + +import Control.Alt ((<|>)) +import Data.Array as Array +import Data.Array.NonEmpty as NonEmptyArray +import Data.Either (Either(..)) +import Data.Map (Map) +import Data.Map as Map +import Data.Maybe (Maybe(..), fromMaybe) +import Data.Newtype (unwrap) +import Data.String as String +import Data.Traversable (traverse) +import Registry.Docgen.Docs (DataConstructorName(..), DocChildDeclaration(..), DocChildDeclarationInfo(..), DocConstraint(..), DocDeclaration(..), DocDeclarationInfo(..), DocModule(..), DocPackage(..), DocReexport(..), DocType(..), ForallBinding(..), FunDep(..), Ident(..), InfixAlias(..), IntLiteral(..), ModuleName(..), OperatorName(..), Qualified(..), Readme, RowLabel(..), RowRep, SourceArtifact, SourceSpan(..), StringLiteral, TypeName(..), TypeVar(..), ValueName(..), isPrim, schemaVersion) +import Registry.Docgen.Legacy.Docs (InPackage(..)) +import Registry.Docgen.Legacy.Docs as L +import Registry.Location (Location(..)) +import Registry.Version as Version +import Safe.Coerce (coerce) + +fromLegacyPackage :: Map ModuleName String -> SourceArtifact -> Maybe Readme -> L.DocPackage -> Either String DocPackage +fromLegacyPackage sourcePaths sourceArtifact readme (L.DocPackage pkg@{ github: L.GithubData github, packageMeta: L.DocPackageMeta meta }) = do + compilerVersion <- Version.parse pkg.compilerVersion + modules <- traverse convertModule pkg.modules + pure $ DocPackage + { schemaVersion + , compilerVersion + , sourceArtifact + , dependencies: meta.dependencies + , description: meta.description + , license: meta.license + , location + , locationRef: Just pkg.versionTag + , name: coerce meta.name + , modules + , readme + , resolvedDependencies: pkg.resolvedDependencies + , resolvedModulePackages: pkg.moduleMap + , version: pkg.version + } + where + convertModule legacy@(L.DocModule { name }) + | isPrim name = Right $ fromLegacyModule sourcePaths legacy + | otherwise = case Map.lookup name sourcePaths of + Nothing -> Left $ "Missing package-relative source path for module " <> unwrap name + Just _ -> Right $ fromLegacyModule sourcePaths legacy + + location :: Location + location = GitHub + { owner: github.user + , repo: github.repo + , subdir: Nothing + } + +fromLegacyModule :: Map ModuleName String -> L.DocModule -> DocModule +fromLegacyModule sourcePaths (L.DocModule mod@{ name: ownerModule }) = + DocModule + { comments: mod.comments + , declarations: fromLegacyDeclaration ownerModule <$> mod.declarations + , name: mod.name + , reexports: fromLegacyReExport <$> mod.reExports + } + where + fromLegacyReExport :: L.ReExport -> DocReexport + fromLegacyReExport (L.ReExport { declarations, moduleName: InPackage { item: moduleName } }) = + DocReexport + { declarations: fromLegacyDeclaration moduleName <$> declarations + , moduleName + } + + fromLegacyDeclaration :: ModuleName -> L.Declaration -> DocDeclaration + fromLegacyDeclaration declarationModule (L.Declaration decl) = + DocDeclaration + { children: fromLegacyChildDeclaration declarationModule <$> decl.children + , info: fromLegacyDeclarationInfo declarationModule decl.title (fromLegacyType declarationModule <<< _.kind <<< unwrap <$> decl.kindInfo) decl.info + , comments: decl.comments + , sourceSpan: replaceSourcePath <$> Map.lookup declarationModule sourcePaths <*> decl.sourceSpan + } + + fromLegacyDeclarationInfo :: ModuleName -> String -> Maybe DocType -> L.DeclarationInfo -> DocDeclarationInfo + fromLegacyDeclarationInfo currentModule title signature = case _ of + L.ValueDeclaration ty -> + DeclValue + { name: Qualified { moduleName: currentModule, name: ValueName title } + , signature: fromLegacyType currentModule ty + } + L.DataDeclaration declType tyVars roles -> + DeclData + { isNewtype: case declType of + L.Newtype -> true + L.Data -> false + , name: Qualified { moduleName: currentModule, name: TypeName title } + , roles + , signature + , vars: fromLegacyTypeVar currentModule <$> tyVars + } + L.TypeSynonymDeclaration tyVars body -> + DeclType + { body: fromLegacyType currentModule body + , name: Qualified { moduleName: currentModule, name: TypeName title } + , signature + , vars: fromLegacyTypeVar currentModule <$> tyVars + } + L.TypeClassDeclaration tyVars cons funDeps -> + DeclTypeClass + { funDeps: fromLegacyFunDep <$> funDeps + , name: Qualified { moduleName: currentModule, name: TypeName title } + , signature + , superClasses: fromLegacyConstraint currentModule <$> cons + , vars: fromLegacyTypeVar currentModule <$> tyVars + } + L.AliasDeclaration (L.Fixity { associativity, precedence }) qual@(L.Qualified _ alias) -> + DeclInfix + { alias: case alias of + Left ty -> + AliasType $ fromLegacyQualified currentModule (qual $> ty) + Right (Left ident) -> + AliasValue $ fromLegacyQualified currentModule (qual $> coerce ident) + Right (Right ctor) -> + AliasConstructor $ fromLegacyQualified currentModule (qual $> ctor) + , associativity + , name: Qualified { moduleName: currentModule, name: unsafeOperatorNameFromTitle title } + , precedence + } + L.ExternDataDeclaration ty roles -> + DeclForeignData + { name: Qualified { moduleName: currentModule, name: TypeName title } + , roles + , signature: fromLegacyType currentModule ty + } + + fromLegacyChildDeclaration :: ModuleName -> L.ChildDeclaration -> DocChildDeclaration + fromLegacyChildDeclaration currentModule (L.ChildDeclaration decl) = + DocChildDeclaration + { comments: decl.comments + , info: fromLegacyChildDeclarationInfo currentModule decl.title decl.info + , sourceSpan: replaceSourcePath <$> Map.lookup currentModule sourcePaths <*> decl.sourceSpan + } + + fromLegacyChildDeclarationInfo :: ModuleName -> String -> L.ChildDeclarationInfo -> DocChildDeclarationInfo + fromLegacyChildDeclarationInfo currentModule title = case _ of + L.ChildInstance cons ty -> + ChildDeclInstance + { constraints: fromLegacyConstraint currentModule <$> cons + , head: fromLegacyType currentModule ty + , name: Qualified { moduleName: currentModule, name: ValueName title } + } + L.ChildDataConstructor tys -> + ChildDeclConstructor + { args: fromLegacyTypePrec currentModule PrecAtom <$> tys + , name: Qualified { moduleName: currentModule, name: DataConstructorName title } + } + L.ChildTypeClassMember ty -> + ChildDeclTypeClassMember + { signature: fromLegacyType currentModule ty + , name: Qualified { moduleName: currentModule, name: ValueName title } + } + + fromLegacyConstraint :: ModuleName -> L.DocConstraint -> DocConstraint + fromLegacyConstraint currentModule (L.DocConstraint name _ args) = + DocConstraint + { args: fromLegacyTypePrec currentModule PrecAtom <$> args + , name: fromLegacyQualified currentModule name + } + + fromLegacyQualified :: forall a. ModuleName -> L.Qualified a -> Qualified a + fromLegacyQualified currentModule (L.Qualified qb name) = case qb of + L.ByModuleName moduleName -> + Qualified { moduleName, name } + L.BySourcePos _ -> + Qualified { moduleName: currentModule, name } + + fromLegacyTypeVar :: ModuleName -> L.DocTypeVar -> TypeVar + fromLegacyTypeVar currentModule (L.DocTypeVar ident signature) = + TypeVar + { ident + , signature: fromLegacyType currentModule <$> signature + } + + fromLegacyFunDep :: L.Fundep -> FunDep + fromLegacyFunDep (L.Fundep determiners determinees) = + FunDep + { determiners + , determinees + } + + fromLegacyTypePrec :: ModuleName -> TypePrec -> L.DocType -> DocType + fromLegacyTypePrec currentModule prec ty = do + let ty' = fromLegacyType currentModule ty + if precOfType ty' <= prec then + ty' + else + TypeParens ty' + + fromLegacyType :: ModuleName -> L.DocType -> DocType + fromLegacyType currentModule = case _ of + L.TypeVar ident -> + TypeIdent $ Ident ident + L.TypeLevelString str -> + TypeString str + L.TypeLevelInt int -> + TypeInt $ IntSmall int + L.TypeWildcard _ -> + TypeWildcard + L.TypeConstructor name -> + TypeConstructor $ fromLegacyQualified currentModule name + L.TypeOp name -> + TypeOperator $ fromLegacyQualified currentModule name + L.TypeApp ty1 ty2 + | Just arg <- isFunction ty1 -> + TypeFunction + { arg: fromLegacyTypePrec currentModule PrecApp arg + , result: fromLegacyTypePrec currentModule PrecArrow ty2 + } + | isRecord ty1 + , L.RCons label ty3 ty4 <- ty2 -> + TypeRecord $ toRowRep currentModule [ toRowLabel currentModule label ty3 ] ty4 + | otherwise -> + TypeApp + { arg: fromLegacyTypePrec currentModule PrecApp ty2 + , function: fromLegacyTypePrec currentModule PrecApp ty1 + } + L.KindApp ty1 ty2 -> + TypeKindApp + { arg: fromLegacyTypePrec currentModule PrecApp ty2 + , function: fromLegacyTypePrec currentModule PrecApp ty1 + } + L.ForAll vis ident sig ty -> + TypeForall $ go (NonEmptyArray.singleton (toForallBinding vis ident sig)) ty + where + toForallBinding vis' ident' sig' = + ForallBinding + { isVisible: case vis' of + L.TypeVarVisible -> true + L.TypeVarInvisible -> false + , name: Ident ident' + , signature: fromLegacyType currentModule <$> sig' + } + go bindings = case _ of + L.ForAll vis' ident' sig' ty' -> + go (NonEmptyArray.snoc bindings (toForallBinding vis' ident' sig')) ty' + body -> + { bindings, body: fromLegacyType currentModule body } + L.ConstrainedType con ty -> + TypeConstrained + { constraint: fromLegacyConstraint currentModule con + , result: fromLegacyTypePrec currentModule PrecArrow ty + } + L.KindedType ty1 ty2 -> + TypeKindSignature + { signature: fromLegacyTypePrec currentModule PrecArrow ty2 + , term: fromLegacyType currentModule ty1 + } + L.BinaryNoParensType ty1 ty2 ty3 -> + TypeInfixApp + { argLhs: fromLegacyTypePrec currentModule PrecApp ty2 + , argRhs: fromLegacyTypePrec currentModule PrecApp ty3 + , operator: fromLegacyType currentModule ty1 + } + L.ParensInType ty -> + TypeParens $ fromLegacyType currentModule ty + L.RCons label ty1 ty2 -> + TypeRow $ toRowRep currentModule [ toRowLabel currentModule label ty1 ] ty2 + L.REmpty -> + TypeRow { labels: [], tail: Nothing } + + toRowLabel :: ModuleName -> StringLiteral -> L.DocType -> RowLabel + toRowLabel currentModule label sig = + RowLabel + { label + , signature: fromLegacyType currentModule sig + } + + toRowRep :: ModuleName -> Array RowLabel -> L.DocType -> RowRep + toRowRep currentModule labels = case _ of + L.RCons label ty1 ty2 -> + toRowRep currentModule (Array.snoc labels (toRowLabel currentModule label ty1)) ty2 + L.REmpty -> + { labels, tail: Nothing } + tail -> + { labels, tail: Just $ fromLegacyType currentModule tail } + +unsafeOperatorNameFromTitle :: String -> OperatorName +unsafeOperatorNameFromTitle title = + OperatorName $ fromMaybe title $ valueOperator <|> typeOperator + where + valueOperator = do + String.stripPrefix (String.Pattern "(") title + >>= String.stripSuffix (String.Pattern ")") + + typeOperator = do + String.stripPrefix (String.Pattern "type (") title + >>= String.stripSuffix (String.Pattern ")") + +replaceSourcePath :: String -> SourceSpan -> SourceSpan +replaceSourcePath path (SourceSpan span) = SourceSpan $ span { path = path } + +data TypePrec + = PrecBottom + | PrecAtom + | PrecApp + | PrecInfix + | PrecArrow + | PrecKinded + | PrecTop + +derive instance Eq TypePrec +derive instance Ord TypePrec + +precOfType :: DocType -> TypePrec +precOfType = case _ of + TypeApp _ -> PrecApp + TypeKindApp _ -> PrecApp + TypeInfixApp _ -> PrecInfix + TypeForall _ -> PrecArrow + TypeConstrained _ -> PrecArrow + TypeFunction _ -> PrecArrow + TypeKindSignature _ -> PrecKinded + _ -> PrecAtom + +isFunction :: L.DocType -> Maybe L.DocType +isFunction = case _ of + L.TypeApp (L.TypeConstructor (L.Qualified (L.ByModuleName (ModuleName "Prim")) (TypeName "Function"))) arg -> + Just arg + _ -> + Nothing + +isRecord :: L.DocType -> Boolean +isRecord = case _ of + L.TypeConstructor (L.Qualified (L.ByModuleName (ModuleName "Prim")) (TypeName "Record")) -> + true + _ -> + false diff --git a/docgen/src/Registry/Docgen/Decoder.purs b/docgen/src/Registry/Docgen/Decoder.purs new file mode 100644 index 000000000..994ed99ae --- /dev/null +++ b/docgen/src/Registry/Docgen/Decoder.purs @@ -0,0 +1,145 @@ +module Registry.Docgen.Decoder where + +import Prelude + +import Codec.JSON.DecodeError (DecodeError) +import Codec.JSON.DecodeError as DecodeError +import Control.Monad.Except (Except, catchError, runExcept, throwError) +import Data.Array as Array +import Data.Codec (Codec') +import Data.Codec as Codec +import Data.Either (Either, either) +import Data.Lazy as Lazy +import Data.Map (Map) +import Data.Map as Map +import Data.Maybe (Maybe(..)) +import Data.Profunctor.Star (Star(..)) +import Data.Traversable (traverse) +import Data.TraversableWithIndex (class TraversableWithIndex, traverseWithIndex) +import Data.Tuple (Tuple(..)) +import JSON (JObject, JSON) +import JSON as JSON +import JSON.Object as JObject +import JSON.Path as JP +import Safe.Coerce (coerce) + +type Decoder a b = Star (Except DecodeError) a b + +defer :: forall a b. (Unit -> Decoder a b) -> Decoder a b +defer k = Star \a -> do + let (Star k') = Lazy.force lazy + k' a + where + lazy = Lazy.defer k + +throw :: forall a b. DecodeError -> Decoder a b +throw err = Star \_ -> throwError err + +liftEither :: forall a b. (a -> Either String b) -> Decoder a b +liftEither fn = Star (either (throwError <<< DecodeError.basic) pure <<< fn) + +runDecoder :: forall a b. Decoder a b -> a -> Either DecodeError b +runDecoder (Star k) = runExcept <<< k + +fromCodec :: forall a b. Codec' (Except DecodeError) a b -> Decoder a b +fromCodec = Star <<< Codec.decode + +toCodec :: forall a b. Decoder a b -> (b -> a) -> Codec' (Except DecodeError) a b +toCodec (Star decoder) = Codec.codec' decoder + +decodeProp :: forall a. String -> Decoder JSON a -> Decoder JObject a +decodeProp prop (Star f) = Star \obj -> + case JObject.lookup prop obj of + Nothing -> + throwError $ DecodeError.noValueFound (JP.AtKey prop JP.Tip) + Just value -> + catchError (f value) \err -> + throwError $ DecodeError.withPath (JP.AtKey prop) err + +decodePropOptional :: forall a. String -> Decoder JSON a -> Decoder JObject (Maybe a) +decodePropOptional prop (Star f) = Star \obj -> + case JObject.lookup prop obj of + Nothing -> + pure Nothing + Just value + | JSON.isNull value -> + pure Nothing + | otherwise -> + catchError (Just <$> f value) \err -> + throwError $ DecodeError.withPath (JP.AtKey prop) err + +decodeOptional :: forall a. Decoder JSON a -> Decoder JSON (Maybe a) +decodeOptional (Star f) = Star \json -> + if JSON.isNull json then + pure Nothing + else + Just <$> f json + +decodeIndex :: forall a. Int -> Decoder JSON a -> Decoder (Array JSON) a +decodeIndex ix (Star f) = Star \arr -> + case Array.index arr ix of + Nothing -> + throwError $ DecodeError.noValueFound (JP.AtIndex ix JP.Tip) + Just value -> + catchError (f value) \err -> + throwError $ DecodeError.withPath (JP.AtIndex ix) err + +decodeJObject :: forall a. Decoder JObject a -> Decoder JSON a +decodeJObject (Star f) = Star \json -> + case JSON.toJObject json of + Nothing -> + throwError $ DecodeError.basic "Expected Object" + Just obj -> + f obj + +decodeMap :: forall k v. Ord k => (String -> Decoder JSON (Tuple k v)) -> Decoder JObject (Map k v) +decodeMap k = Star \obj -> + Map.fromFoldable <$> traverse + ( \(Tuple key value) -> + catchError (coerce (k key) value) \err -> + throwError $ DecodeError.withPath (JP.AtKey key) err + ) + (JObject.entries obj) + +decodeArray :: forall a. Decoder (Array JSON) a -> Decoder JSON a +decodeArray (Star f) = Star \json -> + case JSON.toArray json of + Nothing -> + throwError $ DecodeError.basic "Expected Array" + Just arr -> + f arr + +traversedAtIndex :: forall f a. TraversableWithIndex Int f => Decoder JSON a -> Decoder (f JSON) (f a) +traversedAtIndex (Star f) = Star $ traverseWithIndex \ix value -> + catchError (f value) \err -> + throwError $ DecodeError.withPath (JP.AtIndex ix) err + +guardLength :: Int -> Decoder (Array JSON) Unit +guardLength n = Star \arr -> + if Array.length arr == n then + pure unit + else + throwError $ DecodeError.basic $ "Exected Array of length " <> show n + +decodeString :: Decoder JSON String +decodeString = Star \json -> + case JSON.toString json of + Nothing -> + throwError $ DecodeError.basic "Expected String" + Just str -> + pure str + +decodeInt :: Decoder JSON Int +decodeInt = Star \json -> + case JSON.toInt json of + Nothing -> + throwError $ DecodeError.basic "Expected Int" + Just int -> + pure int + +decodeNull :: Decoder JSON Unit +decodeNull = Star \json -> + if JSON.isNull json then + pure unit + else + throwError $ DecodeError.basic "Expected null" diff --git a/docgen/src/Registry/Docgen/Docs.purs b/docgen/src/Registry/Docgen/Docs.purs new file mode 100644 index 000000000..12eaf28bd --- /dev/null +++ b/docgen/src/Registry/Docgen/Docs.purs @@ -0,0 +1,335 @@ +module Registry.Docgen.Docs where + +import Prelude + +import Data.Array.NonEmpty (NonEmptyArray) +import Data.Map (Map) +import Data.Maybe (Maybe, isJust) +import Data.Newtype (class Newtype) +import Data.String (Pattern(..)) +import Data.String as String +import Registry.License (License) +import Registry.Location (Location) +import Registry.PackageName (PackageName) +import Registry.Sha256 (Sha256) +import Registry.Version (Version) + +schemaVersion :: Int +schemaVersion = 1 + +newtype SourceArtifact = SourceArtifact + { bytes :: Number + , hash :: Sha256 + } + +derive instance Newtype SourceArtifact _ + +newtype RawRange = RawRange String + +derive newtype instance Eq RawRange +derive newtype instance Ord RawRange +derive instance Newtype RawRange _ + +newtype ModuleName = ModuleName String + +derive newtype instance Eq ModuleName +derive newtype instance Ord ModuleName +derive instance Newtype ModuleName _ + +newtype Ident = Ident String + +derive newtype instance Eq Ident +derive newtype instance Ord Ident +derive instance Newtype Ident _ + +newtype TypeName = TypeName String + +derive newtype instance Eq TypeName +derive newtype instance Ord TypeName +derive instance Newtype TypeName _ + +newtype OperatorName = OperatorName String + +derive newtype instance Eq OperatorName +derive newtype instance Ord OperatorName +derive instance Newtype OperatorName _ + +newtype DataConstructorName = DataConstructorName String + +derive newtype instance Eq DataConstructorName +derive newtype instance Ord DataConstructorName +derive instance Newtype DataConstructorName _ + +newtype ValueName = ValueName String + +derive newtype instance Eq ValueName +derive newtype instance Ord ValueName +derive instance Newtype ValueName _ + +newtype Readme = Readme + { content :: String + , extension :: Maybe String + } + +derive instance Newtype Readme _ + +newtype SourcePos = SourcePos + { column :: Int + , line :: Int + } + +derive instance Newtype SourcePos _ + +data Role + = Nominal + | Representational + | Phantom + +newtype SourceSpan = SourceSpan + { end :: SourcePos + , path :: String + , start :: SourcePos + } + +derive instance Newtype SourceSpan _ + +newtype Qualified a = Qualified + { moduleName :: ModuleName + , name :: a + } + +derive instance Newtype (Qualified a) _ + +newtype FunDep = FunDep + { determiners :: Array Ident + , determinees :: Array Ident + } + +derive instance Newtype FunDep _ + +data StringLiteral + = StringValue String + | StringCodeUnits (Array Int) + +data IntLiteral + = IntSmall Int + | IntBig String + +newtype RowLabel = RowLabel + { label :: StringLiteral + , signature :: DocType + } + +derive instance Newtype RowLabel _ + +newtype ForallBinding = ForallBinding + { isVisible :: Boolean + , name :: Ident + , signature :: Maybe DocType + } + +derive instance Newtype ForallBinding _ + +type TypeAppRep = + { arg :: DocType + , function :: DocType + } + +type InfixAppRep = + { argLhs :: DocType + , argRhs :: DocType + , operator :: DocType + } + +type ForallRep = + { bindings :: NonEmptyArray ForallBinding + , body :: DocType + } + +type ConstrainedRep = + { constraint :: DocConstraint + , result :: DocType + } + +type FunctionRep = + { arg :: DocType + , result :: DocType + } + +type KindSignatureRep = + { signature :: DocType + , term :: DocType + } + +type RowRep = + { labels :: Array RowLabel + , tail :: Maybe DocType + } + +data DocType + = TypeWildcard + | TypeIdent Ident + | TypeString StringLiteral + | TypeInt IntLiteral + | TypeConstructor (Qualified TypeName) + | TypeOperator (Qualified OperatorName) + | TypeApp TypeAppRep + | TypeKindApp TypeAppRep + | TypeInfixApp InfixAppRep + | TypeForall ForallRep + | TypeConstrained ConstrainedRep + | TypeFunction FunctionRep + | TypeKindSignature KindSignatureRep + | TypeRecord RowRep + | TypeRow RowRep + | TypeParens DocType + +newtype DocConstraint = DocConstraint + { args :: Array DocType + , name :: Qualified TypeName + } + +derive instance Newtype DocConstraint _ + +type ChildDeclInstanceRep = + { constraints :: Array DocConstraint + , head :: DocType + , name :: Qualified ValueName + } + +type ChildDeclConstructorRep = + { args :: Array DocType + , name :: Qualified DataConstructorName + } + +type ChildDeclTypeClassMemberRep = + { name :: Qualified ValueName + , signature :: DocType + } + +data DocChildDeclarationInfo + = ChildDeclInstance ChildDeclInstanceRep + | ChildDeclConstructor ChildDeclConstructorRep + | ChildDeclTypeClassMember ChildDeclTypeClassMemberRep + +newtype DocChildDeclaration = DocChildDeclaration + { comments :: Maybe String + , info :: DocChildDeclarationInfo + , sourceSpan :: Maybe SourceSpan + } + +derive instance Newtype DocChildDeclaration _ + +newtype TypeVar = TypeVar + { ident :: Ident + , signature :: Maybe DocType + } + +derive instance Newtype TypeVar _ + +type DeclValueRep = + { name :: Qualified ValueName + , signature :: DocType + } + +type DeclDataRep = + { isNewtype :: Boolean + , name :: Qualified TypeName + , roles :: Array Role + , signature :: Maybe DocType + , vars :: Array TypeVar + } + +type DeclTypeRep = + { body :: DocType + , name :: Qualified TypeName + , signature :: Maybe DocType + , vars :: Array TypeVar + } + +type DeclTypeClassRep = + { funDeps :: Array FunDep + , name :: Qualified TypeName + , signature :: Maybe DocType + , superClasses :: Array DocConstraint + , vars :: Array TypeVar + } + +data Associativity + = Infixl + | Infixr + | Infix + +data InfixAlias + = AliasType (Qualified TypeName) + | AliasConstructor (Qualified DataConstructorName) + | AliasValue (Qualified ValueName) + +type DeclInfixRep = + { alias :: InfixAlias + , associativity :: Associativity + , name :: Qualified OperatorName + , precedence :: Int + } + +type DeclForeignDataRep = + { name :: Qualified TypeName + , roles :: Array Role + , signature :: DocType + } + +data DocDeclarationInfo + = DeclValue DeclValueRep + | DeclData DeclDataRep + | DeclType DeclTypeRep + | DeclTypeClass DeclTypeClassRep + | DeclInfix DeclInfixRep + | DeclForeignData DeclForeignDataRep + +newtype DocDeclaration = DocDeclaration + { children :: Array DocChildDeclaration + , info :: DocDeclarationInfo + , comments :: Maybe String + , sourceSpan :: Maybe SourceSpan + } + +derive instance Newtype DocDeclaration _ + +newtype DocReexport = DocReexport + { moduleName :: ModuleName + , declarations :: Array DocDeclaration + } + +derive instance Newtype DocReexport _ + +newtype DocModule = DocModule + { comments :: Maybe String + , declarations :: Array DocDeclaration + , name :: ModuleName + , reexports :: Array DocReexport + } + +derive instance Newtype DocModule _ + +newtype DocPackage = DocPackage + { schemaVersion :: Int + , compilerVersion :: Version + , sourceArtifact :: SourceArtifact + , dependencies :: Map PackageName RawRange + , description :: Maybe String + , license :: License + , location :: Location + , locationRef :: Maybe String + , name :: PackageName + , modules :: Array DocModule + , readme :: Maybe Readme + , resolvedDependencies :: Map PackageName Version + , resolvedModulePackages :: Map ModuleName PackageName + , version :: Version + } + +derive instance Newtype DocPackage _ + +isPrim :: ModuleName -> Boolean +isPrim (ModuleName name) = + name == "Prim" || isJust (String.stripPrefix (Pattern "Prim.") name) diff --git a/docgen/src/Registry/Docgen/Generate.purs b/docgen/src/Registry/Docgen/Generate.purs new file mode 100644 index 000000000..528f9d60f --- /dev/null +++ b/docgen/src/Registry/Docgen/Generate.purs @@ -0,0 +1,137 @@ +module Registry.Docgen.Generate + ( GenerationError(..) + , ModuleInput + , PackageInput + , generatePackage + , printGenerationError + ) where + +import Prelude + +import Data.Array as Array +import Data.Bifunctor (lmap) +import Data.Either (Either(..)) +import Data.Foldable (foldM) +import Data.Map (Map) +import Data.Map as Map +import Data.Maybe (Maybe(..)) +import Data.Newtype (unwrap) +import Data.Set as Set +import Data.String (Pattern(..)) +import Data.String as String +import Data.Tuple (Tuple(..)) +import PureScript.CST.Types as CST +import Registry.Docgen.Convert as Convert +import Registry.Docgen.Docs (DocModule(..), DocPackage(..), ModuleName, RawRange, Readme, SourceArtifact, schemaVersion) +import Registry.Docgen.Legacy.Docs as Legacy +import Registry.Docgen.Reexports (ReexportError) +import Registry.Docgen.Reexports as Reexports +import Registry.License (License) +import Registry.Location (Location) +import Registry.PackageName (PackageName) +import Registry.PackageName as PackageName +import Registry.Version (Version) +import Safe.Coerce (coerce) + +type ModuleInput = + { docs :: Legacy.DocModule + , package :: PackageName + , source :: CST.ModuleHeader Void + , sourcePath :: String + } + +type PackageInput = + { compilerVersion :: Version + , dependencies :: Map PackageName RawRange + , description :: Maybe String + , license :: License + , location :: Location + , locationRef :: Maybe String + , modules :: Array ModuleInput + , name :: PackageName + , readme :: Maybe Readme + , resolvedDependencies :: Map PackageName Version + , sourceArtifact :: SourceArtifact + , version :: Version + } + +data GenerationError + = MismatchedModuleNames ModuleName ModuleName + | DuplicateModule ModuleName + | InvalidSourcePath ModuleName String + | UnknownModulePackage ModuleName PackageName + | NoPackageModules PackageName + | ReexportFailure ReexportError + +printGenerationError :: GenerationError -> String +printGenerationError = case _ of + MismatchedModuleNames docsName sourceName -> + "Compiler docs for module " <> unwrap docsName <> " were paired with source module " <> unwrap sourceName + DuplicateModule name -> + "Received duplicate documentation inputs for module " <> unwrap name + InvalidSourcePath name path -> + "Module " <> unwrap name <> " has a source path that is not relative to the package tarball root: " <> show path + UnknownModulePackage moduleName package -> + "Module " <> unwrap moduleName <> " belongs to " <> PackageName.print package <> ", which is not in the exact package resolutions" + NoPackageModules package -> + "No documentation modules belong to package " <> PackageName.print package + ReexportFailure error -> + Reexports.printReexportError error + +generatePackage :: PackageInput -> Either GenerationError DocPackage +generatePackage input = do + let sortedInputs = Array.sortBy (comparing docsModuleName) input.modules + _ <- foldM validateModule Set.empty sortedInputs + let packageModules = Array.filter (_.package >>> eq input.name) sortedInputs + if Array.null packageModules then + Left $ NoPackageModules input.name + else do + let sourcePaths = Map.fromFoldable $ map (\moduleInput -> Tuple (docsModuleName moduleInput) moduleInput.sourcePath) sortedInputs + let converted = map (Convert.fromLegacyModule sourcePaths <<< _.docs) sortedInputs + resolved <- lmap ReexportFailure $ Reexports.modulesWithReexports converted (map _.source sortedInputs) + let modulePackages = Map.fromFoldable $ map (\moduleInput -> Tuple (docsModuleName moduleInput) moduleInput.package) sortedInputs + pure $ DocPackage + { schemaVersion + , compilerVersion: input.compilerVersion + , sourceArtifact: input.sourceArtifact + , dependencies: input.dependencies + , description: input.description + , license: input.license + , location: input.location + , locationRef: input.locationRef + , modules: Array.filter (\(DocModule { name }) -> Map.lookup name modulePackages == Just input.name) resolved + , name: input.name + , readme: input.readme + , resolvedDependencies: input.resolvedDependencies + , resolvedModulePackages: modulePackages + , version: input.version + } + where + validateModule seen moduleInput = do + let docsName = docsModuleName moduleInput + let sourceName = sourceModuleName moduleInput.source + if docsName /= sourceName then + Left $ MismatchedModuleNames docsName sourceName + else if Set.member docsName seen then + Left $ DuplicateModule docsName + else if not (isPackageRelativePath moduleInput.sourcePath) then + Left $ InvalidSourcePath docsName moduleInput.sourcePath + else if moduleInput.package /= input.name && not (Map.member moduleInput.package input.resolvedDependencies) then + Left $ UnknownModulePackage docsName moduleInput.package + else + Right $ Set.insert docsName seen + +docsModuleName :: ModuleInput -> ModuleName +docsModuleName { docs: Legacy.DocModule { name } } = name + +sourceModuleName :: CST.ModuleHeader Void -> ModuleName +sourceModuleName (CST.ModuleHeader { name: CST.Name { name } }) = coerce name + +isPackageRelativePath :: String -> Boolean +isPackageRelativePath path = + not (String.null path) + && String.take 1 path /= "/" + && not (String.contains (Pattern "\\") path) + && Array.all validPart (String.split (Pattern "/") path) + where + validPart part = not (String.null part) && part /= "." && part /= ".." diff --git a/docgen/src/Registry/Docgen/HTML.purs b/docgen/src/Registry/Docgen/HTML.purs new file mode 100644 index 000000000..6ed4054ce --- /dev/null +++ b/docgen/src/Registry/Docgen/HTML.purs @@ -0,0 +1,196 @@ +module Registry.Docgen.HTML where + +import Prelude hiding (div) + +import Data.Array (fold) +import Data.Foldable (class Foldable, foldMap) +import Data.Newtype (class Newtype, un) +import Data.String (Pattern(..), Replacement(..), replaceAll) + +newtype HTML = HTML String + +derive newtype instance Semigroup HTML +derive newtype instance Monoid HTML +derive instance Newtype HTML _ + +newtype Attribute = Attribute String + +instance Semigroup Attribute where + append (Attribute x) (Attribute y) = case x, y of + "", _ -> Attribute y + _, "" -> Attribute x + _, _ -> Attribute (x <> " " <> y) + +instance Monoid Attribute where + mempty = Attribute "" + +escapeHtml :: String -> String +escapeHtml = + replaceAll (Pattern "&") (Replacement "&") + >>> replaceAll (Pattern "<") (Replacement "<") + >>> replaceAll (Pattern ">") (Replacement ">") + >>> replaceAll (Pattern "\"") (Replacement """) + >>> replaceAll (Pattern "'") (Replacement "'") + +attr :: String -> String -> Attribute +attr attrName attrValue = Attribute $ attrName <> "=\"" <> escapeHtml attrValue <> "\"" + +text :: String -> HTML +text = HTML <<< escapeHtml + +renderTagAttributes :: Array Attribute -> String +renderTagAttributes attrs = case fold attrs of + Attribute "" -> "" + Attribute all -> " " <> all + +renderChildren :: Array HTML -> String +renderChildren = un HTML <<< fold + +element :: String -> Array Attribute -> Array HTML -> HTML +element tagName attrs children = + HTML $ "<" <> tagName <> renderTagAttributes attrs <> ">" <> renderChildren children <> " tagName <> ">" + +leafElement :: String -> Array Attribute -> HTML +leafElement tagName attrs = + HTML $ "<" <> tagName <> renderTagAttributes attrs <> " />" + +div :: Array Attribute -> Array HTML -> HTML +div = element "div" + +span :: Array Attribute -> Array HTML -> HTML +span = element "span" + +p :: Array Attribute -> Array HTML -> HTML +p = element "p" + +pre :: Array Attribute -> Array HTML -> HTML +pre = element "pre" + +code :: Array Attribute -> Array HTML -> HTML +code = element "code" + +section :: Array Attribute -> Array HTML -> HTML +section = element "section" + +article :: Array Attribute -> Array HTML -> HTML +article = element "article" + +main :: Array Attribute -> Array HTML -> HTML +main = element "main" + +h1 :: Array Attribute -> Array HTML -> HTML +h1 = element "h1" + +h2 :: Array Attribute -> Array HTML -> HTML +h2 = element "h2" + +h3 :: Array Attribute -> Array HTML -> HTML +h3 = element "h3" + +h4 :: Array Attribute -> Array HTML -> HTML +h4 = element "h4" + +h5 :: Array Attribute -> Array HTML -> HTML +h5 = element "h5" + +h6 :: Array Attribute -> Array HTML -> HTML +h6 = element "h6" + +ul :: Array Attribute -> Array HTML -> HTML +ul = element "ul" + +ol :: Array Attribute -> Array HTML -> HTML +ol = element "ol" + +li :: Array Attribute -> Array HTML -> HTML +li = element "li" + +a :: Array Attribute -> Array HTML -> HTML +a = element "a" + +button :: Array Attribute -> Array HTML -> HTML +button = element "button" + +form :: Array Attribute -> Array HTML -> HTML +form = element "form" + +img :: Array Attribute -> HTML +img = leafElement "img" + +input :: Array Attribute -> HTML +input = leafElement "input" + +meta :: Array Attribute -> HTML +meta = leafElement "meta" + +link :: Array Attribute -> HTML +link = leafElement "link" + +head :: Array HTML -> HTML +head = element "head" [] + +body :: Array Attribute -> Array HTML -> HTML +body = element "body" + +htmlTitle :: Array HTML -> HTML +htmlTitle = element "title" [] + +wbr :: HTML +wbr = leafElement "wbr" [] + +dl :: Array Attribute -> Array HTML -> HTML +dl = element "dl" + +dt :: Array Attribute -> Array HTML -> HTML +dt = element "dt" + +dd :: Array Attribute -> Array HTML -> HTML +dd = element "dd" + +class_ :: String -> Attribute +class_ = attr "class" + +id :: String -> Attribute +id = attr "id" + +href :: String -> Attribute +href = attr "href" + +lang :: String -> Attribute +lang = attr "lang" + +src :: String -> Attribute +src = attr "src" + +alt :: String -> Attribute +alt = attr "alt" + +type_ :: String -> Attribute +type_ = attr "type" + +name :: String -> Attribute +name = attr "name" + +title :: String -> Attribute +title = attr "title" + +value :: String -> Attribute +value = attr "value" + +rel :: String -> Attribute +rel = attr "rel" + +placeholder :: String -> Attribute +placeholder = attr "placeholder" + +html :: Array Attribute -> Array HTML -> HTML +html = element "html" + +doctype :: HTML +doctype = HTML "\n" + +when :: Boolean -> (Unit -> HTML) -> HTML +when b k = if b then k unit else mempty + +forEach :: forall f a. Foldable f => f a -> (a -> HTML) -> HTML +forEach = flip foldMap diff --git a/docgen/src/Registry/Docgen/Legacy/Docs.purs b/docgen/src/Registry/Docgen/Legacy/Docs.purs new file mode 100644 index 000000000..cf2b23e58 --- /dev/null +++ b/docgen/src/Registry/Docgen/Legacy/Docs.purs @@ -0,0 +1,180 @@ +module Registry.Docgen.Legacy.Docs where + +import Prelude + +import Data.Either (Either) +import Data.Map (Map) +import Data.Maybe (Maybe) +import Data.Newtype (class Newtype) +import Registry.Docgen.Docs (Associativity, DataConstructorName, Ident, ModuleName, OperatorName, RawRange, Role, SourcePos, SourceSpan, StringLiteral, TypeName) +import Registry.License (License) +import Registry.PackageName (PackageName) +import Registry.Version (Version) + +newtype GithubData = GithubData + { user :: String + , repo :: String + } + +derive newtype instance Eq GithubData + +derive newtype instance Ord GithubData + +derive instance Newtype GithubData _ + +data QualifiedBy + = BySourcePos SourcePos + | ByModuleName ModuleName + +data Qualified a = Qualified QualifiedBy a + +derive instance Functor Qualified + +data DataDeclType + = Data + | Newtype + +data DocTypeVar = DocTypeVar Ident (Maybe DocType) + +data Fundep = Fundep (Array Ident) (Array Ident) + +newtype Fixity = Fixity + { associativity :: Associativity + , precedence :: Int + } + +type FixityAlias = Qualified (Either TypeName (Either Ident DataConstructorName)) + +data DeclarationInfo + = ValueDeclaration DocType + | DataDeclaration DataDeclType (Array DocTypeVar) (Array Role) + | TypeSynonymDeclaration (Array DocTypeVar) DocType + | TypeClassDeclaration (Array DocTypeVar) (Array DocConstraint) (Array Fundep) + | AliasDeclaration Fixity FixityAlias + | ExternDataDeclaration DocType (Array Role) + +data ChildDeclarationInfo + = ChildInstance (Array DocConstraint) DocType + | ChildDataConstructor (Array DocType) + | ChildTypeClassMember DocType + +newtype ChildDeclaration = ChildDeclaration + { title :: String + , comments :: Maybe String + , sourceSpan :: Maybe SourceSpan + , info :: ChildDeclarationInfo + } + +derive instance Newtype ChildDeclaration _ + +data KindSignatureFor + = DataSig + | NewtypeSig + | TypeSynonymSig + | ClassSig + +data WildcardData + = HoleWildcard String + | UnnamedWildcard + | IgnoredWildcard + +data TypeVarVisibility + = TypeVarVisible + | TypeVarInvisible + +data DocType + = TypeVar String + | TypeLevelString StringLiteral + | TypeLevelInt Int + | TypeWildcard WildcardData + | TypeConstructor (Qualified TypeName) + | TypeOp (Qualified OperatorName) + | TypeApp DocType DocType + | KindApp DocType DocType + | ForAll TypeVarVisibility String (Maybe DocType) DocType + | ConstrainedType DocConstraint DocType + | KindedType DocType DocType + | BinaryNoParensType DocType DocType DocType + | ParensInType DocType + | RCons StringLiteral DocType DocType + | REmpty + +data DocConstraint = DocConstraint (Qualified TypeName) (Array DocType) (Array DocType) + +newtype KindInfo = KindInfo + { keyword :: KindSignatureFor + , kind :: DocType + } + +derive instance Newtype KindInfo _ + +newtype Declaration = Declaration + { children :: Array ChildDeclaration + , comments :: Maybe String + , info :: DeclarationInfo + , kindInfo :: Maybe KindInfo + , sourceSpan :: Maybe SourceSpan + , title :: String + } + +derive instance Newtype Declaration _ + +newtype InPackage a = InPackage + { item :: a + , package :: Maybe PackageName + } + +derive instance Newtype (InPackage a) _ + +newtype ReExport = ReExport + { declarations :: Array Declaration + , moduleName :: InPackage ModuleName + } + +newtype Author = Author + { name :: String + , email :: Maybe String + , homepage :: Maybe String + } + +derive instance Newtype Author _ + +newtype Repository = Repository + { url :: String + , type :: String + } + +derive instance Newtype Repository _ + +newtype DocModule = DocModule + { comments :: Maybe String + , declarations :: Array Declaration + , name :: ModuleName + , reExports :: Array ReExport + } + +derive instance Newtype DocModule _ + +newtype DocPackageMeta = DocPackageMeta + { authors :: Array Author + , dependencies :: Map PackageName RawRange + , description :: Maybe String + , license :: License + , name :: PackageName + , repository :: Maybe Repository + } + +derive instance Newtype DocPackageMeta _ + +newtype DocPackage = DocPackage + { compilerVersion :: String + , github :: GithubData + , moduleMap :: Map ModuleName PackageName + , modules :: Array DocModule + , packageMeta :: DocPackageMeta + , resolvedDependencies :: Map PackageName Version + , version :: Version + , versionTag :: String + } + +derive instance Newtype DocPackage _ diff --git a/docgen/src/Registry/Docgen/Legacy/JSON.purs b/docgen/src/Registry/Docgen/Legacy/JSON.purs new file mode 100644 index 000000000..58664994e --- /dev/null +++ b/docgen/src/Registry/Docgen/Legacy/JSON.purs @@ -0,0 +1,512 @@ +module Registry.Docgen.Legacy.JSON + ( decodeDocPackage + , decodeDocModule + ) where + +import Prelude + +import Codec.JSON.DecodeError (DecodeError) +import Codec.JSON.DecodeError as DecodeError +import Control.Alt ((<|>)) +import Data.Array as Array +import Data.Array.NonEmpty as NonEmptyArray +import Data.Either (Either(..)) +import Data.Map (Map) +import Data.Map as Map +import Data.Maybe (Maybe(..), fromMaybe) +import Data.Tuple (Tuple(..)) +import JSON (JObject, JSON) +import JSON as JSON +import JSON.Path as JP +import Registry.Docgen.Decoder (Decoder, decodeArray, decodeIndex, decodeInt, decodeJObject, decodeMap, decodeNull, decodeOptional, decodeProp, decodePropOptional, decodeString, defer, guardLength, liftEither, runDecoder, throw, traversedAtIndex) +import Registry.Docgen.Docs (Associativity(..), DataConstructorName(..), Ident(..), ModuleName(..), OperatorName(..), RawRange(..), Role(..), SourcePos(..), SourceSpan(..), StringLiteral(..), TypeName(..)) +import Registry.Docgen.Legacy.Docs (Author(..), ChildDeclaration(..), ChildDeclarationInfo(..), DataDeclType(..), Declaration(..), DeclarationInfo(..), DocConstraint(..), DocModule(..), DocPackage(..), DocPackageMeta(..), DocType(..), DocTypeVar(..), Fixity(..), FixityAlias, Fundep(..), GithubData(..), InPackage(..), KindInfo(..), KindSignatureFor(..), Qualified(..), QualifiedBy(..), ReExport(..), Repository(..), TypeVarVisibility(..), WildcardData(..)) +import Registry.License (License) +import Registry.License as License +import Registry.PackageName (PackageName) +import Registry.PackageName as PackageName +import Registry.Version (Version) +import Registry.Version as Version +import Safe.Coerce (coerce) + +decodeDocPackage :: JSON -> Either DecodeError DocPackage +decodeDocPackage = runDecoder decodeDocPackage' + +decodeDocModule :: JSON -> Either DecodeError DocModule +decodeDocModule = runDecoder decodeDocModule' + +decodeDocPackage' :: Decoder JSON DocPackage +decodeDocPackage' = decodeJObject ado + compilerVersion <- decodeProp "compilerVersion" decodeString + github <- decodeProp "github" decodeGithubData + moduleMap <- decodeProp "moduleMap" decodeModuleMap + modules <- decodeProp "modules" $ decodeArray (traversedAtIndex decodeDocModule') + packageMeta <- decodeProp "packageMeta" decodePackageMeta + resolvedDependencies <- decodePropOptional "resolvedDependencies" $ decodeDependencies decodeVersion + version <- decodeProp "version" decodeVersion + versionTag <- decodeProp "versionTag" decodeString + in + DocPackage + { compilerVersion + , github + , moduleMap + , modules + , packageMeta + , resolvedDependencies: fromMaybe Map.empty resolvedDependencies + , version + , versionTag + } + +decodePackageName :: Decoder JSON PackageName +decodePackageName = map fixName decodeString >>> liftEither PackageName.parse + where + fixName name + | Array.elem name allowedPrefixNames = name + | otherwise = PackageName.stripPureScriptPrefix name + + allowedPrefixNames = + [ "purescript-compiler-backend-utilities" + ] + +decodeModuleMap :: Decoder JSON (Map ModuleName PackageName) +decodeModuleMap = decodeJObject $ decodeMap \key -> + Tuple (ModuleName key) <$> decodePackageName + +decodeGithubData :: Decoder JSON GithubData +decodeGithubData = decodeArray ado + user <- decodeIndex 0 decodeString + repo <- decodeIndex 1 decodeString + in GithubData { user, repo } + +decodeVersion :: Decoder JSON Version +decodeVersion = liftEither Version.parse <<< decodeString + +decodePackageMeta :: Decoder JSON DocPackageMeta +decodePackageMeta = decodeJObject ado + authors <- decodePropOptional "authors" $ decodeArray (traversedAtIndex decodeAuthor) + dependencies <- decodePropOptional "dependencies" $ decodeDependencies decodeRange + description <- decodePropOptional "description" decodeString + license <- decodeProp "license" decodeLicense + name <- decodeProp "name" decodePackageName + repository <- decodePropOptional "repository" decodeRepository + in + DocPackageMeta + { authors: fromMaybe [] authors + , dependencies: fromMaybe Map.empty dependencies + , description + , license + , name + , repository + } + +decodeLicense :: Decoder JSON License +decodeLicense = stringLicense <|> arrayLicense + where + stringLicense = + liftEither License.parse <<< decodeString + + arrayLicense = ado + licenses <- decodeArray $ traversedAtIndex stringLicense >>> liftEither toNonEmpty + in License.joinWith License.Or licenses + + toNonEmpty licenses = case NonEmptyArray.fromArray licenses of + Just nonEmpty -> Right nonEmpty + Nothing -> Left "Expected non-empty license array" + +decodeRepository :: Decoder JSON Repository +decodeRepository = decodeJObject ado + url <- decodeProp "url" decodeString + type_ <- decodeProp "type" decodeString + in Repository { url, type: type_ } + +decodeAuthor :: Decoder JSON Author +decodeAuthor = decodeStringAuthor <|> decodeObjectAuthor + where + decodeStringAuthor = + Author <<< { name: _, email: Nothing, homepage: Nothing } <$> decodeString + + decodeObjectAuthor = decodeJObject ado + name <- decodeProp "name" decodeString + email <- decodePropOptional "email" decodeString + homepage <- decodePropOptional "homepage" decodeString + in Author { name, email, homepage } + +decodeDependencies :: forall a. Decoder JSON a -> Decoder JSON (Map PackageName a) +decodeDependencies decoder = decodeJObject $ decodeMap \key -> + Tuple <$> (pure (JSON.fromString key) >>> decodePackageName) <*> decoder + +decodeRange :: Decoder JSON RawRange +decodeRange = RawRange <$> decodeString + +decodeDocModule' :: Decoder JSON DocModule +decodeDocModule' = decodeJObject ado + comments <- decodePropOptional "comments" decodeString + declarations <- decodeProp "declarations" $ decodeArray (traversedAtIndex decodeDeclaration) + name <- decodeProp "name" decodeString + reExports <- decodeProp "reExports" $ decodeArray (traversedAtIndex decodeReExport) + in + DocModule + { comments + , declarations + , name: ModuleName name + , reExports + } + +decodeDeclaration :: Decoder JSON Declaration +decodeDeclaration = defer \_ -> decodeJObject ado + children <- decodeProp "children" $ decodeArray (traversedAtIndex decodeChildDeclaration) + comments <- decodePropOptional "comments" decodeString + info <- decodeProp "info" decodeDeclarationInfo + kindInfo <- decodePropOptional "kind" decodeKindInfo + sourceSpan <- decodePropOptional "sourceSpan" decodeSourceSpan + title <- decodeProp "title" decodeString + in + Declaration + { children + , comments + , info + , kindInfo + , sourceSpan + , title + } + +decodeKindInfo :: Decoder JSON KindInfo +decodeKindInfo = decodeJObject ado + keyword <- decodeProp "keyword" decodeKindSignatureFor + kind <- decodeProp "kind" decodeDocType + in KindInfo { keyword, kind } + +decodeKindSignatureFor :: Decoder JSON KindSignatureFor +decodeKindSignatureFor = decodeString >>= case _ of + "data" -> pure DataSig + "newtype" -> pure NewtypeSig + "class" -> pure ClassSig + "type" -> pure TypeSynonymSig + other -> throw $ DecodeError.basic $ "Unexpected KindSignatureFor: " <> other + +decodeChildDeclaration :: Decoder JSON ChildDeclaration +decodeChildDeclaration = decodeJObject ado + comments <- decodePropOptional "comments" decodeString + info <- decodeProp "info" decodeChildDeclarationInfo + sourceSpan <- decodePropOptional "sourceSpan" decodeSourceSpan + title <- decodeProp "title" decodeString + in ChildDeclaration { comments, info, sourceSpan, title } + +decodeChildDeclarationInfo :: Decoder JSON ChildDeclarationInfo +decodeChildDeclarationInfo = decodeJObject do + declType <- decodeProp "declType" decodeString + case declType of + "instance" -> decodeInstance + "dataConstructor" -> decodeDataConstructor + "typeClassMember" -> decodeTypeClassMember + other -> throw $ DecodeError.error (JP.AtKey "declType" JP.Tip) $ "Unexpected value: " <> other + where + decodeInstance = ado + dependencies <- decodeProp "dependencies" $ decodeArray (traversedAtIndex decodeDocConstraint) + type_ <- decodeProp "type" decodeDocType + in ChildInstance dependencies type_ + + decodeDataConstructor = ado + arguments <- decodeProp "arguments" $ decodeArray (traversedAtIndex decodeDocType) + in ChildDataConstructor arguments + + decodeTypeClassMember = ado + type_ <- decodeProp "type" decodeDocType + in ChildTypeClassMember type_ + +decodeDeclarationInfo :: Decoder JSON DeclarationInfo +decodeDeclarationInfo = decodeJObject do + declType <- decodeProp "declType" decodeString + case declType of + "value" -> decodeValueDeclaration + "data" -> decodeDataDeclaration + "externData" -> decodeExternData + "typeSynonym" -> decodeTypeSynonym + "typeClass" -> decodeTypeClass + "alias" -> decodeAlias + "kind" -> decodeCompatKind + _ -> throw $ DecodeError.error (JP.AtKey "declType" JP.Tip) $ "Unexpected value: " <> declType + where + decodeValueDeclaration = ado + type_ <- decodeProp "type" decodeDocType + in ValueDeclaration type_ + + decodeDataDeclaration = ado + dataDeclType <- decodeProp "dataDeclType" decodeDataDeclType + typeArguments <- decodeProp "typeArguments" $ decodeArray (traversedAtIndex decodeTypeVar) + roles <- decodePropOptional "roles" $ decodeArray (traversedAtIndex decodeRole) + in DataDeclaration dataDeclType typeArguments (fromMaybe [] roles) + + decodeExternData = ado + kind <- decodeProp "kind" decodeDocType + roles <- decodePropOptional "roles" $ decodeArray (traversedAtIndex decodeRole) + in ExternDataDeclaration kind (fromMaybe [] roles) + + decodeTypeSynonym = ado + arguments <- decodeProp "arguments" $ decodeArray (traversedAtIndex decodeTypeVar) + type_ <- decodeProp "type" decodeDocType + in TypeSynonymDeclaration arguments type_ + + decodeTypeClass = ado + arguments <- decodeProp "arguments" $ decodeArray (traversedAtIndex decodeTypeVar) + superclasses <- decodeProp "superclasses" $ decodeArray (traversedAtIndex decodeDocConstraint) + fundeps <- decodePropOptional "fundeps" $ decodeArray (traversedAtIndex decodeFundep) + in TypeClassDeclaration arguments superclasses (fromMaybe [] fundeps) + + decodeAlias = ado + fixity <- decodeProp "fixity" decodeFixity + alias <- decodeProp "alias" decodeFixityAlias + in AliasDeclaration fixity alias + + decodeCompatKind = + pure $ ExternDataDeclaration (TypeConstructor primType) [] + + primType = Qualified (ByModuleName (ModuleName "Prim")) (TypeName "Type") + +decodeFixityAlias :: Decoder JSON FixityAlias +decodeFixityAlias = decodeQualified $ decodeEither decodeTypeName (decodeEither decodeIdent decodeDataConstructorName) + +decodeTypeName :: Decoder JSON TypeName +decodeTypeName = TypeName <$> decodeString + +decodeDataConstructorName :: Decoder JSON DataConstructorName +decodeDataConstructorName = DataConstructorName <$> decodeString + +decodeIdent :: Decoder JSON Ident +decodeIdent = decodeJObject $ decodeProp "Ident" (Ident <$> decodeString) + +decodeEither :: forall a b. Decoder JSON a -> Decoder JSON b -> Decoder JSON (Either a b) +decodeEither left right = decodeJObject do + Left <$> decodeProp "Left" left + <|> Right <$> decodeProp "Right" right + +decodeFixity :: Decoder JSON Fixity +decodeFixity = decodeJObject ado + associativity <- decodeProp "associativity" decodeAssociativity + precedence <- decodeProp "precedence" decodeInt + in Fixity { associativity, precedence } + +decodeAssociativity :: Decoder JSON Associativity +decodeAssociativity = decodeString >>= case _ of + "infixl" -> pure Infixl + "infixr" -> pure Infixr + "infix" -> pure Infix + other -> throw $ DecodeError.basic $ "Unexpected associativity: " <> other + +decodeFundep :: Decoder JSON Fundep +decodeFundep = decodeArray ado + a <- decodeIndex 0 $ decodeArray (traversedAtIndex decodeString) + b <- decodeIndex 1 $ decodeArray (traversedAtIndex decodeString) + in Fundep (coerce a) (coerce b) + +decodeTypeVar :: Decoder JSON DocTypeVar +decodeTypeVar = decodeArray ado + name <- decodeIndex 0 decodeString + type_ <- decodeIndex 1 $ decodeOptional decodeDocType + in DocTypeVar (Ident name) type_ + +decodeRole :: Decoder JSON Role +decodeRole = decodeString >>= case _ of + "Representational" -> pure Representational + "Nominal" -> pure Nominal + "Phantom" -> pure Phantom + other -> throw $ DecodeError.basic $ "Unexpected Role: " <> other + +decodeDataDeclType :: Decoder JSON DataDeclType +decodeDataDeclType = decodeString >>= case _ of + "data" -> pure Data + "newtype" -> pure Newtype + other -> throw $ DecodeError.basic $ "Unexpected DataDeclType: " <> other + +decodeSourceSpan :: Decoder JSON SourceSpan +decodeSourceSpan = decodeJObject ado + end <- decodeProp "end" decodeSourcePos + path <- decodeProp "name" decodeString + start <- decodeProp "start" decodeSourcePos + in SourceSpan { end, path, start } + +decodeReExport :: Decoder JSON ReExport +decodeReExport = defer \_ -> decodeJObject ado + moduleName <- decodeProp "moduleName" $ decodeInPackage decodeModuleName + declarations <- decodeProp "declarations" $ decodeArray (traversedAtIndex decodeDeclaration) + in ReExport { moduleName, declarations } + +decodeInPackage :: forall a. Decoder JSON a -> Decoder JSON (InPackage a) +decodeInPackage decoder = decodeJObject ado + package <- decodePropOptional "package" decodePackageName + item <- decodeProp "item" decoder + in InPackage { package, item } + +decodeDocType :: Decoder JSON DocType +decodeDocType = defer \_ -> + decodeJObject do + tag <- decodeProp "tag" decodeString + case tag of + "TypeVar" -> decodeTypeVar' + "TypeLevelString" -> decodeTypeLevelString + "TypeLevelInt" -> decodeTypeLevelInt + "TypeWildcard" -> decodeTypeWildcard + "TypeConstructor" -> decodeTypeConstructor + "TypeOp" -> decodeTypeOp + "TypeApp" -> decodeTypeApp + "KindApp" -> decodeKindApp + "ForAll" -> decodeForAll + "ConstrainedType" -> decodeConstrainedType + "KindedType" -> decodeKindedType + "BinaryNoParensType" -> decodeBinaryNoParensType + "ParensInType" -> decodeParensInType + "REmpty" -> pure REmpty + "RCons" -> decodeRCons + "Row" -> decodeCompatRow + "FunKind" -> decodeCompatFunKind + "NamedKind" -> decodeCompatNamedKind + _ -> throw $ DecodeError.error (JP.AtKey "tag" JP.Tip) $ "Unexpected value: " <> tag + where + decodeTypeVar' = + decodeContents $ TypeVar <$> decodeString + + decodeTypeLevelString = + decodeContents $ TypeLevelString <$> decodePSString + + decodeTypeLevelInt = + decodeContents $ TypeLevelInt <$> decodeInt + + decodeTypeWildcard = + decodeContents $ TypeWildcard <$> decodeWildcardData + + decodeTypeConstructor = + decodeContents $ TypeConstructor <<< coerce <$> decodeQualified decodeString + + decodeTypeOp = + decodeContents $ TypeOp <<< coerce <$> decodeQualified decodeString + + decodeTypeApp = defer \_ -> + decodeContents $ decodeArray ado + lhs <- decodeIndex 0 decodeDocType + rhs <- decodeIndex 1 decodeDocType + in TypeApp lhs rhs + + decodeKindApp = defer \_ -> + decodeContents $ decodeArray ado + lhs <- decodeIndex 0 decodeDocType + rhs <- decodeIndex 1 decodeDocType + in KindApp lhs rhs + + decodeForAll = defer \_ -> do + let + asObject = decodeJObject ado + visibility <- decodeProp "visibility" decodeTypeVarVisibility + identifier <- decodeProp "identifier" decodeString + kind <- decodePropOptional "kind" decodeDocType + type_ <- decodeProp "type" decodeDocType + in ForAll visibility identifier kind type_ + + withoutKind = decodeArray do + guardLength 3 + identifier <- decodeIndex 0 decodeString + type_ <- decodeIndex 1 decodeDocType + pure $ ForAll TypeVarInvisible identifier Nothing type_ + + withKind = decodeArray do + guardLength 4 + identifier <- decodeIndex 0 decodeString + kind <- decodeIndex 1 $ decodeOptional decodeDocType + type_ <- decodeIndex 2 decodeDocType + pure $ ForAll TypeVarInvisible identifier kind type_ + + decodeContents $ asObject <|> withoutKind <|> withKind + + decodeConstrainedType = defer \_ -> + decodeContents $ decodeArray ado + constraint <- decodeIndex 0 decodeDocConstraint + type_ <- decodeIndex 1 decodeDocType + in ConstrainedType constraint type_ + + decodeKindedType = defer \_ -> + decodeContents $ decodeArray ado + a <- decodeIndex 0 decodeDocType + b <- decodeIndex 1 decodeDocType + in KindedType a b + + decodeBinaryNoParensType = defer \_ -> + decodeContents $ decodeArray ado + a <- decodeIndex 0 decodeDocType + b <- decodeIndex 1 decodeDocType + c <- decodeIndex 2 decodeDocType + in BinaryNoParensType a b c + + decodeParensInType = defer \_ -> + decodeContents $ ParensInType <$> decodeDocType + + decodeRCons = defer \_ -> + decodeContents $ decodeArray ado + label <- decodeIndex 0 decodePSString + type_ <- decodeIndex 1 decodeDocType + tail <- decodeIndex 2 decodeDocType + in RCons label type_ tail + + decodeCompatRow = defer \_ -> + decodeContents ado + type_ <- decodeDocType + in TypeApp (TypeConstructor primRow) type_ + + decodeCompatFunKind = defer \_ -> + decodeContents $ decodeArray ado + a <- decodeIndex 0 decodeDocType + b <- decodeIndex 1 decodeDocType + in TypeApp (TypeApp (TypeConstructor primFunction) a) b + + decodeCompatNamedKind = + decodeContents $ TypeConstructor <<< coerce <$> decodeQualified decodeString + + primRow = Qualified (ByModuleName (ModuleName "Prim")) (TypeName "Row") + primFunction = Qualified (ByModuleName (ModuleName "Prim")) (TypeName "Function") + +decodeContents :: forall a. Decoder JSON a -> Decoder JObject a +decodeContents = decodeProp "contents" + +decodeDocConstraint :: Decoder JSON DocConstraint +decodeDocConstraint = defer \_ -> + decodeJObject ado + class_ <- decodeProp "constraintClass" $ decodeQualified decodeString + kindArgs <- decodePropOptional "constraintKindArgs" $ decodeArray (traversedAtIndex decodeDocType) + args <- decodeProp "constraintArgs" $ decodeArray (traversedAtIndex decodeDocType) + in DocConstraint (coerce class_) (fromMaybe [] kindArgs) args + +decodeTypeVarVisibility :: Decoder JSON TypeVarVisibility +decodeTypeVarVisibility = + decodeString >>= case _ of + "TypeVarVisible" -> pure TypeVarVisible + "TypeVarInvisible" -> pure TypeVarInvisible + _ -> throw $ DecodeError.basic "Expected TypeVarVisibility" + +decodeQualified :: forall a. Decoder JSON a -> Decoder JSON (Qualified a) +decodeQualified decoder = decodeArray ado + qual <- decodeIndex 0 decodeQualifiedBy + value <- decodeIndex 1 decoder + in Qualified qual value + +decodeModuleName :: Decoder JSON ModuleName +decodeModuleName = ModuleName <<< Array.intercalate "." <$> decodeArray (traversedAtIndex decodeString) + +decodeQualifiedBy :: Decoder JSON QualifiedBy +decodeQualifiedBy = + ByModuleName <$> decodeModuleName + <|> BySourcePos <$> decodeSourcePos + <|> BySourcePos (SourcePos { line: 0, column: 0 }) <$ decodeNull + +decodeSourcePos :: Decoder JSON SourcePos +decodeSourcePos = decodeArray ado + line <- decodeIndex 0 decodeInt + column <- decodeIndex 1 decodeInt + in SourcePos { line, column } + +decodeWildcardData :: Decoder JSON WildcardData +decodeWildcardData = + HoleWildcard <$> decodeString + <|> pure UnnamedWildcard + +decodePSString :: Decoder JSON StringLiteral +decodePSString = + StringValue <$> decodeString + <|> StringCodeUnits <$> decodeArray (traversedAtIndex decodeInt) diff --git a/docgen/src/Registry/Docgen/Package/Render.purs b/docgen/src/Registry/Docgen/Package/Render.purs new file mode 100644 index 000000000..c3a607d28 --- /dev/null +++ b/docgen/src/Registry/Docgen/Package/Render.purs @@ -0,0 +1,550 @@ +module Registry.Docgen.Package.Render where + +import Prelude + +import Data.Array as Array +import Data.Array.NonEmpty (NonEmptyArray) +import Data.Array.NonEmpty as NonEmptyArray +import Data.Foldable (fold, foldMap, intercalate) +import Data.Map (Map) +import Data.Map as Map +import Data.Maybe (Maybe(..), fromMaybe) +import Data.Newtype (unwrap) +import Data.String (Pattern(..)) +import Data.String as String +import Data.String.CodeUnits as SCU +import Data.String.Regex as Regex +import Data.String.Regex.Flags (noFlags) +import Data.String.Regex.Unsafe (unsafeRegex) +import Data.Tuple (Tuple(..)) +import Registry.Docgen.Commonmark (renderMarkdownHTML) +import Registry.Docgen.Docs (DataConstructorName(..), DocChildDeclaration(..), DocChildDeclarationInfo(..), DocDeclaration(..), DocDeclarationInfo(..), DocModule(..), DocPackage(..), DocReexport(..), InfixAlias(..), ModuleName, OperatorName(..), Qualified(..), RawRange(..), Readme(..), SourcePos(..), SourceSpan(..), TypeName(..), ValueName(..), isPrim) +import Registry.Docgen.HTML (HTML) +import Registry.Docgen.HTML as H +import Registry.Docgen.Package.Render.Code (CodeRenderer, renderChildDeclConstructor, renderChildDeclInstance, renderChildDeclTypeClassMember, renderDeclData, renderDeclForeignData, renderDeclInfix, renderDeclType, renderDeclTypeClass, renderDeclValue) +import Registry.Docgen.Package.Types (ModuleRef(..), Namespace(..)) +import Registry.License as License +import Registry.Location (Location) +import Registry.Location as Location +import Registry.PackageName (PackageName) +import Registry.PackageName as PackageName +import Registry.Version as Version + +type Link = + { href :: String + , title :: String + } + +type PackageLinker = + { getModuleLink :: ModuleName -> Link + , getPackageLink :: PackageName -> Link + , getRefLink :: { moduleName :: ModuleName, namespace :: Namespace, ref :: String } -> Link + , getSourceLink :: { moduleName :: ModuleName, sourceSpan :: SourceSpan } -> Link + } + +type RenderedDeclInfo = + { anchorId :: String + , content :: HTML + , title :: String + } + +type RenderedChildDecl = + { anchorId :: String + , comments :: Maybe String + , content :: HTML + , title :: String + } + +renderModuleName :: ModuleName -> HTML +renderModuleName = + unwrap + >>> String.split (Pattern ".") + >>> map H.text + >>> intercalate (H.text "." <> H.wbr) + +namespaceAnchor :: Namespace -> String +namespaceAnchor = case _ of + NSType -> "t:" + NSValue -> "v:" + +defaultPackageLinker :: DocPackage -> PackageLinker +defaultPackageLinker + ( DocPackage + { modules + , name: packageName + , resolvedDependencies + , resolvedModulePackages + , version + } + ) = + { getModuleLink, getPackageLink, getRefLink, getSourceLink } + where + versionMap = + Map.insert packageName version resolvedDependencies + + moduleMap = + Map.union resolvedModulePackages + $ Map.fromFoldable + $ map (\(DocModule { name }) -> Tuple name packageName) modules + + getModuleLink moduleName + | isPrim moduleName = + { href: intercalate "/" + [ "" + , "builtins" + , "docs" + , unwrap moduleName + ] + , title: unwrap moduleName + } + | otherwise = fold do + resolvedPackageName <- Map.lookup moduleName moduleMap + let { href, title } = getPackageLink resolvedPackageName + let printedModuleName = unwrap moduleName + pure + { href: intercalate "/" + [ href + , "docs" + , printedModuleName + ] + , title: title <> " " <> printedModuleName + } + + getPackageLink name = fold do + resolvedVersion <- Map.lookup name versionMap + let printedName = PackageName.print name + let printedVersion = Version.print resolvedVersion + pure + { href: intercalate "/" + [ "" + , "packages" + , printedName + , printedVersion + ] + , title: printedName <> "@" <> printedVersion + } + + getRefLink { moduleName, namespace, ref } = do + let { href, title } = getModuleLink moduleName + { href: href <> "#" <> namespaceAnchor namespace <> ref + , title: title <> "." <> ref + } + + getSourceLink + { moduleName + , sourceSpan: SourceSpan + { path + , start: SourcePos { line: startLine } + , end: SourcePos { line: endLine } + } + } = fold do + resolvedPackageName <- Map.lookup moduleName moduleMap + resolvedVersion <- Map.lookup resolvedPackageName versionMap + let printedPackageName = PackageName.print resolvedPackageName + let printedVersion = Version.print resolvedVersion + pure + { href: intercalate "/" + [ "https://www.purescript.org/registry-package-viewer/#" + , printedPackageName + , printedVersion + , path <> "#" <> show startLine <> "-" <> show endLine + ] + , title: printedPackageName <> "@" <> printedVersion <> "/" <> path + } + +htmlCodeRenderer :: PackageLinker -> ModuleName -> CodeRenderer HTML +htmlCodeRenderer { getRefLink } currentModule = + { keyword: \str -> + H.span + [ H.class_ "keyword" ] + [ H.text str ] + , label: \str -> do + let unquoted = SCU.drop 1 (SCU.dropRight 1 str) + H.span + [ H.class_ "row-label" ] + [ H.text if isIdent unquoted then unquoted else str ] + , line: \indent children -> + H.div + [ H.class_ $ "indent-" <> show indent ] + [ children ] + , reference: \mod ref -> do + let + label = + H.span + [ H.class_ if isIdent ref then "ident" else "ctor" ] + [ H.text ref ] + case mod of + Local -> + label + ModuleRef moduleName namespace -> do + let + { href, title } = getRefLink + { moduleName: fromMaybe currentModule moduleName + , namespace + , ref + } + H.a + [ H.href href, H.title title ] + [ label ] + , role: \str -> + H.span + [ H.class_ "role" ] + [ H.text str ] + , space: + H.text " " + , syntax: \str -> + H.span + [ H.class_ "syntax" ] + [ H.text str ] + } + where + isIdent = + Regex.test $ unsafeRegex "^[a-z_][a-zA-Z0-9_']*" noFlags + +renderDocument :: { body :: HTML, title :: String } -> HTML +renderDocument { body, title } = + H.html [ H.lang "en" ] + [ H.head + [ H.htmlTitle + [ H.text title ] + , H.link + [ H.href "https://fonts.googleapis.com/css?family=Roboto+Mono|Roboto:300,400,400i,700,700i" + , H.type_ "text/css" + , H.rel "stylesheet" + ] + , H.link + [ H.href "https://pursuit.purescript.org/static/res/css/normalize.css?etag=fKzu1nci" + , H.type_ "text/css" + , H.rel "stylesheet" + ] + , H.link + [ H.href "https://pursuit.purescript.org/static/res/css/pursuit.css?etag=5eIKlitR" + , H.type_ "text/css" + , H.rel "stylesheet" + ] + , H.link + [ H.href "https://pursuit.purescript.org/static/res/css/extra.css?etag=d4aey1o-" + , H.type_ "text/css" + , H.rel "stylesheet" + ] + ] + , H.body [] [ body ] + ] + +renderContainer :: { anchorId :: String, content :: HTML } -> HTML +renderContainer { anchorId, content } = + H.main + [ H.class_ "container clearfix" + , H.id anchorId + ] + [ content ] + +renderPackageIndex :: PackageLinker -> DocPackage -> HTML +renderPackageIndex linker (DocPackage { dependencies, license, location, modules, name, readme }) = + pageLayout + { label: H.text "Package" + , title: H.text $ PackageName.print name + , main: fold + [ groupedList + [ Tuple (H.text "Repository") + [ renderRepository location + ] + , Tuple (H.text "License") + [ H.text $ License.print license ] + ] + , H.forEach readme \(Readme { content, extension }) -> do + case foldMap String.toLower extension of + "md" -> + H.article + [ H.class_ "readme readme--markdown" ] + [ renderMarkdownHTML { safe: true } content ] + _ -> + H.article + [ H.class_ "readme readme--plain" ] + [ H.text content ] + ] + , sidebar: fold + [ renderModuleList linker modules + , renderDependencyList linker dependencies + ] + } + +renderModule :: PackageLinker -> DocPackage -> DocModule -> HTML +renderModule + linker + (DocPackage { location, modules, name: packageName }) + ( DocModule + { comments + , declarations + , name: moduleName + , reexports + } + ) = + pageLayout + { label: H.text "module" + , title: H.text $ unwrap moduleName + , main: fold + [ groupedList + [ Tuple (H.text "Package") + [ H.text $ PackageName.print packageName ] + , Tuple (H.text "Repository") + [ renderRepository location ] + ] + , H.forEach comments $ renderMarkdownHTML { safe: true } + , H.forEach declarations $ renderDeclaration linker code moduleName + , H.forEach reexports $ renderReexport linker code + ] + , sidebar: renderModuleList linker modules + } + where + code :: CodeRenderer HTML + code = htmlCodeRenderer linker moduleName + +renderDeclaration :: PackageLinker -> CodeRenderer HTML -> ModuleName -> DocDeclaration -> HTML +renderDeclaration { getSourceLink } code moduleName (DocDeclaration { children, info, comments, sourceSpan }) = do + let { anchorId, content, title } = renderDeclarationInfo code info + H.div + [ H.class_ "decl" + , H.id anchorId + ] + [ H.h3 + [ H.class_ "decl__title clearfix" ] + [ H.a + [ H.class_ "decl__anchor" + , H.href $ "#" <> anchorId + ] + [ H.text "#" ] + , H.span [] + [ H.text title ] + , H.forEach sourceSpan \span -> do + let { href, title } = getSourceLink { moduleName, sourceSpan: span } + H.a + [ H.class_ "decl__source" + , H.href href + , H.title title + ] + [ H.text "Source" ] + ] + , H.div + [ H.class_ "decl__body" ] + [ H.pre + [ H.class_ "decl__signature" ] + [ H.code [] + [ content ] + ] + , H.forEach comments $ renderMarkdownHTML { safe: true } + , H.forEach (renderChildConstructors code children) \items -> fold + [ H.h4 [] + [ H.text "Constructors" ] + , H.ul [] + [ H.forEach items renderChildDeclaration ] + ] + , H.forEach (renderChildMembers code children) \items -> fold + [ H.h4 [] + [ H.text "Members" ] + , H.ul [] + [ H.forEach items renderChildDeclaration ] + ] + , H.forEach (renderChildInstances code children) \items -> fold + [ H.h4 [] + [ H.text "Instances" ] + , H.ul [] + [ H.forEach items renderChildDeclaration ] + ] + ] + ] + +renderReexport :: PackageLinker -> CodeRenderer HTML -> DocReexport -> HTML +renderReexport linker@{ getModuleLink } code (DocReexport { declarations, moduleName }) = do + let { href, title } = getModuleLink moduleName + fold + [ H.h2 + [ H.class_ "reexports" ] + [ H.text "Re-exports from " + , H.a + [ H.href href + , H.title title + ] + [ H.text $ unwrap moduleName ] + ] + , H.forEach declarations $ renderDeclaration linker code moduleName + ] + +renderDeclarationInfo :: CodeRenderer HTML -> DocDeclarationInfo -> RenderedDeclInfo +renderDeclarationInfo code = case _ of + DeclValue rep -> do + let (Qualified { name: ValueName title }) = rep.name + { anchorId: namespaceAnchor NSValue <> title + , content: renderDeclValue code rep + , title + } + DeclData rep -> do + let (Qualified { name: TypeName title }) = rep.name + { anchorId: namespaceAnchor NSType <> title + , content: renderDeclData code rep + , title + } + DeclType rep -> do + let (Qualified { name: TypeName title }) = rep.name + { anchorId: namespaceAnchor NSType <> title + , content: renderDeclType code rep + , title + } + DeclTypeClass rep -> do + let (Qualified { name: TypeName title }) = rep.name + { anchorId: namespaceAnchor NSType <> title + , content: renderDeclTypeClass code rep + , title + } + DeclInfix rep -> do + let + (Qualified { name: OperatorName title }) = rep.name + namespace = case rep.alias of + AliasType _ -> NSType + _ -> NSValue + { anchorId: namespaceAnchor namespace <> title + , content: renderDeclInfix code rep + , title + } + DeclForeignData rep -> do + let (Qualified { name: TypeName title }) = rep.name + { anchorId: namespaceAnchor NSType <> title + , content: renderDeclForeignData code rep + , title + } + +renderChildDeclaration :: RenderedChildDecl -> HTML +renderChildDeclaration childProps = + H.li + [ H.id childProps.anchorId ] + [ childProps.content + , H.forEach childProps.comments \md -> + H.div + [ H.class_ "decl__child_comments" ] + [ renderMarkdownHTML { safe: true } md ] + ] + +renderChildConstructors :: CodeRenderer HTML -> Array DocChildDeclaration -> Maybe (NonEmptyArray RenderedChildDecl) +renderChildConstructors code = NonEmptyArray.fromArray <<< Array.mapMaybe case _ of + DocChildDeclaration { comments, info: ChildDeclConstructor rep } -> do + let (Qualified { name: DataConstructorName title }) = rep.name + let anchorId = namespaceAnchor NSValue <> title + Just + { anchorId + , comments + , content: H.code [] [ renderChildDeclConstructor code rep ] + , title + } + _ -> + Nothing + +renderChildInstances :: CodeRenderer HTML -> Array DocChildDeclaration -> Maybe (NonEmptyArray RenderedChildDecl) +renderChildInstances code = NonEmptyArray.fromArray <<< Array.mapMaybe case _ of + DocChildDeclaration { comments, info: ChildDeclInstance rep } -> do + let (Qualified { name: ValueName title }) = rep.name + let anchorId = namespaceAnchor NSValue <> title + Just + { anchorId + , comments + , content: H.code [] [ renderChildDeclInstance code rep ] + , title + } + _ -> + Nothing + +renderChildMembers :: CodeRenderer HTML -> Array DocChildDeclaration -> Maybe (NonEmptyArray RenderedChildDecl) +renderChildMembers code = NonEmptyArray.fromArray <<< Array.mapMaybe case _ of + DocChildDeclaration { comments, info: ChildDeclTypeClassMember rep } -> do + let (Qualified { name: ValueName title }) = rep.name + let anchorId = namespaceAnchor NSValue <> title + Just + { anchorId + , comments + , content: H.code [] [ renderChildDeclTypeClassMember code rep ] + , title + } + _ -> + Nothing + +renderRepository :: Location -> HTML +renderRepository = case _ of + Location.GitHub { owner, repo } -> + H.a + [ H.href $ "https://github.com/" <> owner <> "/" <> repo ] + [ H.text $ "github.com/" <> owner <> "/" <> repo ] + Location.Git { url } -> + H.a + [ H.href url ] + [ H.text url ] + +renderModuleList :: PackageLinker -> Array DocModule -> HTML +renderModuleList { getModuleLink } modules = + groupedList + [ Tuple (H.text "Modules") $ + renderModuleLink <$> modules + ] + where + renderModuleLink (DocModule { name }) = do + let { href, title } = getModuleLink name + H.a + [ H.href href + , H.title title + ] + [ renderModuleName name ] + +renderDependencyList :: PackageLinker -> Map PackageName RawRange -> HTML +renderDependencyList { getPackageLink } dependencies = + groupedList + [ Tuple (H.text "Dependencies") $ + renderDependencyLink <$> Map.toUnfoldable dependencies + ] + where + renderDependencyLink (Tuple packageName (RawRange range)) = do + let { href, title } = getPackageLink packageName + H.div + [ H.class_ "deplink" ] + [ H.a + [ H.class_ "deplink__link" + , H.href href + , H.title title + ] + [ H.text $ PackageName.print packageName ] + , H.span + [ H.class_ "deplink__version" ] + [ H.text range ] + ] + +pageLayout :: { label :: HTML, main :: HTML, sidebar :: HTML, title :: HTML } -> HTML +pageLayout { label, main, sidebar, title } = fold + [ H.div + [ H.class_ "page-title clearfix" ] + [ H.div + [ H.class_ "page-title__label" ] + [ label ] + , H.h1 + [ H.class_ "page-title__title" ] + [ title ] + ] + , H.div + [ H.class_ "col col--main" ] + [ main ] + , H.div + [ H.class_ "col col--aside" ] + [ sidebar ] + ] + +groupedList :: Array (Tuple HTML (Array HTML)) -> HTML +groupedList items = + H.dl + [ H.class_ "grouped-list" ] + (renderItem <$> items) + where + renderItem (Tuple title children) = fold + [ H.dt + [ H.class_ "grouped-list__title" ] + [ title ] + , H.forEach children \item -> + H.dd + [ H.class_ "grouped-list__item" ] + [ item ] + ] diff --git a/docgen/src/Registry/Docgen/Package/Render/Code.purs b/docgen/src/Registry/Docgen/Package/Render/Code.purs new file mode 100644 index 000000000..13b1e5d28 --- /dev/null +++ b/docgen/src/Registry/Docgen/Package/Render/Code.purs @@ -0,0 +1,335 @@ +module Registry.Docgen.Package.Render.Code where + +import Prelude + +import Data.Array (intercalate) +import Data.Array as Array +import Data.Char (fromCharCode) +import Data.Foldable (fold, foldMap, foldl) +import Data.Maybe (Maybe(..)) +import Data.Monoid (guard) +import Data.Newtype (class Newtype, unwrap) +import Data.Semigroup.Foldable (intercalateMap) +import Data.String.CodeUnits (fromCharArray) +import Registry.Docgen.Docs (Associativity(..), ChildDeclConstructorRep, ChildDeclInstanceRep, ChildDeclTypeClassMemberRep, DeclDataRep, DeclForeignDataRep, DeclInfixRep, DeclTypeClassRep, DeclTypeRep, DeclValueRep, DocConstraint(..), DocType(..), ForallBinding(..), FunDep(..), Ident(..), InfixAlias(..), IntLiteral(..), Qualified(..), Role(..), RowLabel(..), RowRep, StringLiteral(..), TypeName, TypeVar(..)) +import Registry.Docgen.Package.Types (ModuleRef(..), Namespace(..)) + +type CodeRenderer rep = + { keyword :: String -> rep + , label :: String -> rep + , line :: Int -> rep -> rep + , reference :: ModuleRef -> String -> rep + , role :: String -> rep + , space :: rep + , syntax :: String -> rep + } + +renderType :: forall rep. Monoid rep => CodeRenderer rep -> DocType -> rep +renderType code@{ keyword, reference, space, syntax } = render + where + render :: DocType -> rep + render = case _ of + TypeWildcard -> + syntax "_" + TypeIdent ident -> + reference Local (unwrap ident) + TypeString str -> + syntax $ renderStringLiteral str + TypeInt n -> + renderIntLiteral n + TypeConstructor name -> + renderQualified code NSType name + TypeOperator name -> + renderQualified code NSType name + TypeApp { function, arg } -> + render function + <> space + <> render arg + TypeKindApp { function, arg } -> + render function + <> space + <> syntax "@" + <> render arg + TypeInfixApp { argLhs, operator: TypeOperator name, argRhs } -> + render argLhs + <> space + <> renderQualified code NSType name + <> space + <> render argRhs + TypeInfixApp { argLhs, operator, argRhs } -> + render argLhs + <> space + <> syntax "`" + <> render operator + <> syntax "`" + <> space + <> render argRhs + TypeForall { bindings, body } -> + keyword "forall" + <> space + <> intercalateMap space renderBinding bindings + <> syntax "." + <> space + <> render body + TypeConstrained { constraint, result } -> + render (constraintAsType constraint) + <> space + <> syntax "=>" + <> space + <> render result + TypeFunction { arg, result } -> + render arg + <> space + <> syntax "->" + <> space + <> render result + TypeKindSignature { term, signature } -> + render term + <> space + <> syntax "::" + <> space + <> render signature + TypeRecord { labels: [], tail: Nothing } -> + syntax "{}" + TypeRecord row -> + syntax "{" + <> space + <> renderRow row + <> space + <> syntax "}" + TypeRow { labels: [], tail: Nothing } -> + syntax "()" + TypeRow row -> + syntax "(" + <> renderRow row + <> syntax ")" + TypeParens ty -> + syntax "(" + <> render ty + <> syntax ")" + + renderRow :: RowRep -> rep + renderRow { labels, tail } = + intercalate (syntax "," <> space) (renderLabel code <$> labels) + <> foldMap (\ty -> space <> syntax "|" <> space <> render ty) tail + + renderBinding :: ForallBinding -> rep + renderBinding (ForallBinding { isVisible, name, signature }) = case signature of + Just ty -> + syntax "(" + <> variable + <> space + <> syntax "::" + <> space + <> render ty + Nothing -> + variable + where + variable = + guard isVisible (syntax "@") + <> reference Local (unwrap name) + + renderIntLiteral :: IntLiteral -> rep + renderIntLiteral = case _ of + IntSmall n -> + syntax $ show n + IntBig n -> + syntax n + +renderQualified :: forall rep a. Newtype a String => CodeRenderer rep -> Namespace -> Qualified a -> rep +renderQualified { reference } namespace (Qualified { moduleName, name }) = + reference (ModuleRef (Just moduleName) namespace) $ unwrap name + +renderLabel :: forall rep. Monoid rep => CodeRenderer rep -> RowLabel -> rep +renderLabel code@{ label, space, syntax } (RowLabel { label: lbl, signature }) = + label (renderStringLiteral lbl) + <> space + <> syntax "::" + <> space + <> renderType code signature + +renderStringLiteral :: StringLiteral -> String +renderStringLiteral = case _ of + StringValue str -> + show str + StringCodeUnits arr -> + show $ fromCharArray $ Array.mapMaybe fromCharCode arr + +constraintAsType :: DocConstraint -> DocType +constraintAsType (DocConstraint { args, name }) = + foldl (\function arg -> TypeApp { function, arg }) (TypeConstructor name) args + +renderTypeVar :: forall rep. Monoid rep => CodeRenderer rep -> TypeVar -> rep +renderTypeVar code@{ reference, space, syntax } (TypeVar { ident: Ident name, signature }) = case signature of + Nothing -> + reference Local name + Just kind -> fold + [ syntax "(" + , reference Local name + , space + , syntax "::" + , space + , renderType code kind + , syntax ")" + ] + +renderTypeVarsWithRoles :: forall rep. Monoid rep => CodeRenderer rep -> Array TypeVar -> Array Role -> rep +renderTypeVarsWithRoles code@{ space } vars roles = + intercalate space $ Array.zipWith (\var r -> renderTypeVar code var <> renderRole code r) vars roles + +renderRole :: forall rep. Monoid rep => CodeRenderer rep -> Role -> rep +renderRole { role } = case _ of + Representational -> mempty + Nominal -> role "nominal" + Phantom -> role "phantom" + +renderConstraintContext :: forall rep. Monoid rep => CodeRenderer rep -> Array (DocConstraint) -> rep -> rep +renderConstraintContext code@{ syntax, space } constraints arrow + | Array.null constraints = mempty + | otherwise = fold + [ syntax "(" + , intercalate (syntax "," <> space) $ + map (renderType code <<< constraintAsType) constraints + , syntax ")" + , space + , arrow + , space + ] + +renderDeclSignature :: forall rep. Monoid rep => CodeRenderer rep -> String -> Qualified TypeName -> DocType -> rep +renderDeclSignature code@{ keyword, space, syntax } kw name signature = + intercalate space + [ keyword kw + , renderQualified code NSType name + , syntax "::" + , renderType code signature + ] + +renderDeclValue :: forall rep. Monoid rep => CodeRenderer rep -> DeclValueRep -> rep +renderDeclValue code@{ space, syntax } { name, signature } = + intercalate space + [ renderQualified code NSValue name + , syntax "::" + , renderType code signature + ] + +renderDeclData :: forall rep. Monoid rep => CodeRenderer rep -> DeclDataRep -> rep +renderDeclData code@{ keyword, line, space } { isNewtype, name, roles, signature, vars } = + fold + [ foldMap (line 0 <<< renderDeclSignature code dataKw name) signature + , line 0 $ keyword dataKw + <> space + <> renderQualified code NSType name + <> renderTypeVarsWithRoles code vars roles + ] + where + dataKw + | isNewtype = "newtype" + | otherwise = "data" + +renderDeclForeignData :: forall rep. Monoid rep => CodeRenderer rep -> DeclForeignDataRep -> rep +renderDeclForeignData code@{ line } { name, roles, signature } = fold + [ line 0 $ renderDeclSignature code "foreign import data" name signature + , line 0 $ renderTypeVarsWithRoles code [] roles + ] + +renderDeclType :: forall rep. Monoid rep => CodeRenderer rep -> DeclTypeRep -> rep +renderDeclType code@{ keyword, line, space, syntax } { body, name, signature, vars } = + fold + [ foldMap (line 0 <<< renderDeclSignature code "type" name) signature + , line 0 $ intercalate space + [ keyword "type" + , renderQualified code NSType name + , intercalate space (renderTypeVar code <$> vars) + , syntax "=" + , renderType code body + ] + ] + +renderDeclTypeClass :: forall rep. Monoid rep => CodeRenderer rep -> DeclTypeClassRep -> rep +renderDeclTypeClass code@{ keyword, line, reference, space, syntax } { funDeps, name, signature, superClasses, vars } = + fold + [ foldMap (line 0 <<< renderDeclSignature code "class" name) signature + , line 0 $ fold + [ keyword "class" + , space + , renderConstraintContext code superClasses (syntax "<=") + , renderQualified code NSType name + , space + , intercalate space (renderTypeVar code <$> vars) + , renderedFunDeps + ] + ] + where + renderedFunDeps + | Array.null funDeps = mempty + | otherwise = fold + [ space + , syntax "|" + , space + , intercalate (syntax "," <> space) $ renderFunDep <$> funDeps + ] + + renderFunDep (FunDep { determinees, determiners }) = + fold + [ intercalate space (reference Local <<< unwrap <$> determiners) + , space + , syntax "->" + , space + , intercalate space (reference Local <<< unwrap <$> determinees) + ] + +renderDeclInfix :: forall rep. Monoid rep => CodeRenderer rep -> DeclInfixRep -> rep +renderDeclInfix code@{ keyword, space, syntax } { alias, associativity, name, precedence } = + fold + [ keyword infixKw + , space + , syntax (show precedence) + , typeKw + , space + , aliasName + , space + , keyword "as" + , space + , renderQualified code (if isTypeOp then NSType else NSValue) name + ] + + where + isTypeOp = case alias of + AliasType _ -> true + _ -> false + + infixKw = case associativity of + Infix -> "infix" + Infixl -> "infixl" + Infixr -> "infixr" + + typeKw = + guard isTypeOp $ space <> keyword "type" + + aliasName = case alias of + AliasType ty -> + renderQualified code NSType ty + AliasConstructor ctor -> + renderQualified code NSValue ctor + AliasValue val -> + renderQualified code NSValue val + +renderChildDeclConstructor :: forall rep. Monoid rep => CodeRenderer rep -> ChildDeclConstructorRep -> rep +renderChildDeclConstructor code@{ space } { args, name } = + foldl go (renderQualified code NSValue name) args + where + go acc arg = acc <> space <> renderType code arg + +renderChildDeclInstance :: forall rep. Monoid rep => CodeRenderer rep -> ChildDeclInstanceRep -> rep +renderChildDeclInstance code@{ syntax } { constraints, head } = + renderConstraintContext code constraints (syntax "=>") + <> renderType code head + +renderChildDeclTypeClassMember :: forall rep. Monoid rep => CodeRenderer rep -> ChildDeclTypeClassMemberRep -> rep +renderChildDeclTypeClassMember code@{ space, syntax } { name, signature } = + intercalate space + [ renderQualified code NSValue name + , syntax "::" + , renderType code signature + ] diff --git a/docgen/src/Registry/Docgen/Package/Types.purs b/docgen/src/Registry/Docgen/Package/Types.purs new file mode 100644 index 000000000..d6de2020f --- /dev/null +++ b/docgen/src/Registry/Docgen/Package/Types.purs @@ -0,0 +1,35 @@ +module Registry.Docgen.Package.Types where + +import Data.Maybe (Maybe) +import Data.Version (Version) +import Registry.Docgen.Docs (ModuleName) +import Registry.PackageName (PackageName) + +-- | References point to either a Type or Value namespace. +data Namespace + = NSType + | NSValue + +-- | Linkable references need to point a module and namespace. Local references +-- | Have nothing interesting to link to. A `Nothing` value for the module name +-- | is considered to be the "current" module being processed. +data ModuleRef + = ModuleRef (Maybe ModuleName) Namespace + | Local + +type ExactPackage = + { packageName :: PackageName + , version :: Version + } + +type ExactModule = + { moduleName :: ModuleName + , packageName :: PackageName + , version :: Version + } + +type ExactRef = + { module :: ExactModule + , namespace :: Namespace + , ref :: String + } diff --git a/docgen/src/Registry/Docgen/Reexports.purs b/docgen/src/Registry/Docgen/Reexports.purs new file mode 100644 index 000000000..84c1fe142 --- /dev/null +++ b/docgen/src/Registry/Docgen/Reexports.purs @@ -0,0 +1,428 @@ +module Registry.Docgen.Reexports + ( modulesWithReexports + , ReexportError(..) + , printReexportError + ) where + +import Prelude + +import Control.Bind (bindFlipped) +import Data.Array as Array +import Data.Array.NonEmpty (NonEmptyArray) +import Data.Array.NonEmpty as NonEmptyArray +import Data.Either (Either(..)) +import Data.Foldable (fold, foldMap, foldl) +import Data.Map (Map, SemigroupMap(..)) +import Data.Map as Map +import Data.Maybe (Maybe(..), fromMaybe, maybe) +import Data.Newtype (un) +import Data.Set (Set) +import Data.Set as Set +import Data.Traversable (traverse) +import Data.Tuple (Tuple(..), fst, snd) +import PureScript.CST.Types as CST +import Registry.Docgen.Docs (DataConstructorName(..), DocChildDeclaration(..), DocChildDeclarationInfo(..), DocConstraint(..), DocDeclaration(..), DocDeclarationInfo(..), DocModule(..), DocReexport(..), DocType(..), ForallBinding(..), InfixAlias(..), ModuleName, OperatorName(..), Qualified(..), TypeName(..), TypeVar(..), ValueName(..), isPrim) +import Safe.Coerce (coerce) + +data RefSet a = RefAll | RefSet a | RefHiding a + +instance Semigroup (RefSet ModuleMemberSet) where + append = case _, _ of + RefSet a, RefSet b -> RefSet (a <> b) + RefHiding hidden, RefSet selected -> RefHiding (differenceMemberSet hidden selected) + RefSet selected, RefHiding hidden -> RefHiding (differenceMemberSet hidden selected) + RefHiding a, RefHiding b -> RefHiding (intersectMemberSet a b) + RefAll, _ -> RefAll + _, RefAll -> RefAll + +newtype ModuleMemberSet = ModuleMemberSet + { operators :: Set OperatorName + , typeClasses :: Set TypeName + , typeOperators :: Set OperatorName + , types :: Map TypeName (RefSet (Set DataConstructorName)) + , values :: Set ValueName + } + +instance Semigroup ModuleMemberSet where + append (ModuleMemberSet a) (ModuleMemberSet b) = ModuleMemberSet + { operators: a.operators <> b.operators + , typeClasses: a.typeClasses <> b.typeClasses + , typeOperators: a.typeOperators <> b.typeOperators + , types: Map.unionWith mergeConstructorRefs a.types b.types + , values: a.values <> b.values + } + +mergeConstructorRefs :: RefSet (Set DataConstructorName) -> RefSet (Set DataConstructorName) -> RefSet (Set DataConstructorName) +mergeConstructorRefs = case _, _ of + RefSet a, RefSet b -> RefSet (a <> b) + RefAll, _ -> RefAll + _, RefAll -> RefAll + RefHiding a, RefHiding b -> RefHiding $ Set.intersection a b + RefHiding hidden, RefSet selected -> RefHiding $ Set.difference hidden selected + RefSet selected, RefHiding hidden -> RefHiding $ Set.difference hidden selected + +differenceConstructorRefs + :: RefSet (Set DataConstructorName) + -> RefSet (Set DataConstructorName) + -> Maybe (RefSet (Set DataConstructorName)) +differenceConstructorRefs = case _, _ of + RefSet a, RefSet b -> fromSet $ Set.difference a b + RefSet _, RefAll -> Nothing + RefSet a, RefHiding b -> fromSet $ Set.intersection a b + RefAll, RefSet b -> Just $ RefHiding b + RefAll, RefAll -> Nothing + RefAll, RefHiding b -> fromSet b + RefHiding a, RefSet b -> Just $ RefHiding $ a <> b + RefHiding _, RefAll -> Nothing + RefHiding a, RefHiding b -> fromSet $ Set.difference b a + where + fromSet set + | Set.isEmpty set = Nothing + | otherwise = Just $ RefSet set + +instance Monoid ModuleMemberSet where + mempty = ModuleMemberSet + { operators: mempty + , typeClasses: mempty + , typeOperators: mempty + , types: Map.empty + , values: mempty + } + +differenceMemberSet :: ModuleMemberSet -> ModuleMemberSet -> ModuleMemberSet +differenceMemberSet (ModuleMemberSet a) (ModuleMemberSet b) = ModuleMemberSet + { operators: Set.difference a.operators b.operators + , typeClasses: Set.difference a.typeClasses b.typeClasses + , typeOperators: Set.difference a.typeOperators b.typeOperators + , types: foldl subtractType a.types (Map.toUnfoldable b.types :: Array (Tuple TypeName (RefSet (Set DataConstructorName)))) + , values: Set.difference a.values b.values + } + where + subtractType types (Tuple name selected) = case Map.lookup name types >>= flip differenceConstructorRefs selected of + Nothing -> Map.delete name types + Just remaining -> Map.insert name remaining types + +intersectMemberSet :: ModuleMemberSet -> ModuleMemberSet -> ModuleMemberSet +intersectMemberSet (ModuleMemberSet a) (ModuleMemberSet b) = ModuleMemberSet + { operators: Set.intersection a.operators b.operators + , typeClasses: Set.intersection a.typeClasses b.typeClasses + , typeOperators: Set.intersection a.typeOperators b.typeOperators + , types: Map.intersectionWith intersectConstructors a.types b.types + , values: Set.intersection a.values b.values + } + where + intersectConstructors = case _, _ of + RefAll, refs -> refs + refs, RefAll -> refs + RefSet x, RefSet y -> RefSet $ Set.intersection x y + RefHiding x, RefHiding y -> RefHiding $ x <> y + RefHiding hidden, RefSet selected -> RefSet $ Set.difference selected hidden + RefSet selected, RefHiding hidden -> RefSet $ Set.difference selected hidden + +type ImportModuleSet = SemigroupMap ModuleName (RefSet ModuleMemberSet) + +data ReexportError + = MissingSourceHeaders (NonEmptyArray ModuleName) + | MissingDocsTarget ModuleName ModuleName + | ReexportCycle (NonEmptyArray ModuleName) + | TransitivelyBlocked ModuleName ModuleName ReexportError + +printReexportError :: ReexportError -> String +printReexportError = case _ of + MissingSourceHeaders names -> + "Missing parsed source headers for documentation modules: " <> printChain names + MissingDocsTarget owner dependency -> + "Module " <> printName owner <> " reexports missing documentation module " <> printName dependency + ReexportCycle names -> + "Reexport cycle: " <> printChain names + TransitivelyBlocked owner dependency cause -> + "Module " <> printName owner <> " is blocked by reexport " <> printName dependency <> ": " <> printReexportError cause + where + printName = coerce + printChain = NonEmptyArray.toArray >>> map printName >>> Array.intercalate " -> " + +modulesWithReexports :: Array DocModule -> Array (CST.ModuleHeader Void) -> Either ReexportError (Array DocModule) +modulesWithReexports allDocs allSourceModules = do + case NonEmptyArray.fromArray missingHeaders of + Just missing -> Left $ MissingSourceHeaders missing + Nothing -> traverse (resolve []) allDocs + where + sourceModulesByName = Map.fromFoldable $ map + ( \(sourceModule@(CST.ModuleHeader { name: CST.Name { name } })) -> + Tuple (coerce name) sourceModule + ) + allSourceModules + + docsByName = Map.fromFoldable $ map (\doc@(DocModule { name }) -> Tuple name doc) allDocs + + missingHeaders = Array.mapMaybe + (\(DocModule { name }) -> if isPrim name || Map.member name sourceModulesByName then Nothing else Just name) + allDocs + + importsFor name + | isPrim name = Map.empty + | otherwise = maybe Map.empty (Map.delete name <<< reexportsOf) $ Map.lookup name sourceModulesByName + + resolve chain (DocModule docs) + | Array.elem docs.name chain = + Left $ ReexportCycle $ NonEmptyArray.cons' docs.name (Array.drop 1 (Array.dropWhile (_ /= docs.name) chain) <> [ docs.name ]) + | otherwise = + do + reexports <- traverse resolveDependency $ Map.toUnfoldable $ importsFor docs.name + pure $ DocModule docs { reexports = mergeReexports $ Array.concat reexports } + where + resolveDependency (Tuple dependency refs) = case Map.lookup dependency docsByName of + Nothing -> Left $ MissingDocsTarget docs.name dependency + Just dependencyDocs -> case resolve (Array.snoc chain docs.name) dependencyDocs of + Left err@(ReexportCycle _) -> Left err + Left err -> Left $ TransitivelyBlocked docs.name dependency err + Right resolved -> Right $ map + (\(Tuple moduleName declarations) -> DocReexport { moduleName, declarations: NonEmptyArray.toArray declarations }) + (matchingExportsOf refs resolved) + +matchingExportsOf :: RefSet ModuleMemberSet -> DocModule -> Array (Tuple ModuleName (NonEmptyArray DocDeclaration)) +matchingExportsOf refSet (DocModule { declarations, name, reexports }) = + groupByModuleName case refSet of + RefAll -> allDecls + RefSet ms -> traverse (declInMemberSet ms) =<< allDecls + RefHiding hidden -> bindFlipped (\(Tuple moduleName decl) -> map (Tuple moduleName) $ declNotInMemberSet hidden decl) allDecls + where + allDecls = + map (Tuple name) declarations <> + bindFlipped (\(DocReexport re) -> map (Tuple re.moduleName) re.declarations) reexports + + groupByModuleName = + Array.groupAllBy (comparing fst) >>> map \decls -> do + let (Tuple moduleName _) = NonEmptyArray.head decls + Tuple moduleName (snd <$> decls) + +declInMemberSet :: ModuleMemberSet -> DocDeclaration -> Array DocDeclaration +declInMemberSet (ModuleMemberSet members) decl@(DocDeclaration { children, info, comments, sourceSpan }) = case info of + DeclValue { name } + | Set.member (unqualify name) members.values -> + pure decl + DeclData { name } + | Just ctorsRef <- Map.lookup (unqualify name) members.types -> + pure $ DocDeclaration + { children: case ctorsRef of + RefAll -> children + RefSet ctors -> + Array.filter + case _ of + DocChildDeclaration { info: ChildDeclConstructor { name: ctorName } } -> + Set.member (unqualify ctorName) ctors + _ -> + true + children + RefHiding ctors -> + Array.filter + case _ of + DocChildDeclaration { info: ChildDeclConstructor { name: ctorName } } -> + not $ Set.member (unqualify ctorName) ctors + _ -> + true + children + , info + , comments + , sourceSpan + } + DeclForeignData { name } + | Map.member (unqualify name) members.types -> + pure decl + DeclType { name } + | Map.member (unqualify name) members.types -> + pure decl + DeclTypeClass { name, vars } + | Set.member (unqualify name) members.typeClasses -> + pure $ DocDeclaration + { children: Array.filter + case _ of + DocChildDeclaration { info: ChildDeclTypeClassMember { name: childName } } -> + Set.member (unqualify childName) members.values + _ -> + true + children + , info + , comments + , sourceSpan + } + | otherwise -> + Array.mapMaybe + case _ of + DocChildDeclaration + { comments: childComments + , info: ChildDeclTypeClassMember { name: childName, signature } + , sourceSpan: childSourceSpan + } + | Set.member (unqualify childName) members.values -> + Just $ DocDeclaration + { children: [] + , info: DeclValue + { name: childName + , signature: promotedMemberSignature name vars signature + } + , comments: childComments + , sourceSpan: childSourceSpan + } + _ -> + Nothing + children + DeclInfix { name, alias: AliasConstructor _ } + | Set.member (unqualify name) members.operators -> + pure decl + DeclInfix { name, alias: AliasValue _ } + | Set.member (unqualify name) members.operators -> + pure decl + DeclInfix { name, alias: AliasType _ } + | Set.member (unqualify name) members.typeOperators -> + pure decl + _ -> + [] + where + unqualify :: forall a. Qualified a -> a + unqualify (Qualified { name }) = name + +declNotInMemberSet :: ModuleMemberSet -> DocDeclaration -> Array DocDeclaration +declNotInMemberSet (ModuleMemberSet members) decl@(DocDeclaration declaration@{ children, info }) = case info of + DeclData { name } -> case Map.lookup (unqualify name) members.types of + Nothing -> pure decl + Just RefAll -> [] + Just (RefSet hidden) + | Set.isEmpty hidden -> [] + | otherwise -> pure $ DocDeclaration declaration { children = filterConstructors (not <<< flip Set.member hidden) children } + Just (RefHiding visible) -> + pure $ DocDeclaration declaration { children = filterConstructors (flip Set.member visible) children } + DeclTypeClass { name } + | Set.member (unqualify name) members.typeClasses -> [] + | otherwise -> pure $ DocDeclaration declaration + { children = Array.filter + case _ of + DocChildDeclaration { info: ChildDeclTypeClassMember { name: childName } } -> + not $ Set.member (unqualify childName) members.values + _ -> true + declaration.children + } + _ + | Array.null $ declInMemberSet (ModuleMemberSet members) decl -> pure decl + | otherwise -> [] + where + filterConstructors keep = Array.filter case _ of + DocChildDeclaration { info: ChildDeclConstructor { name } } -> keep $ unqualify name + _ -> true + + unqualify :: forall a. Qualified a -> a + unqualify (Qualified { name }) = name + +promotedMemberSignature :: Qualified TypeName -> Array TypeVar -> DocType -> DocType +promotedMemberSignature className vars signature = + maybe constrained (\bindings -> TypeForall { bindings: map toBinding bindings, body: constrained }) $ NonEmptyArray.fromArray vars + where + constrained = TypeConstrained + { constraint: DocConstraint + { args: map (\(TypeVar { ident }) -> TypeIdent ident) vars + , name: className + } + , result: signature + } + toBinding (TypeVar { ident, signature: kind }) = ForallBinding { isVisible: false, name: ident, signature: kind } + +mergeReexports :: Array DocReexport -> Array DocReexport +mergeReexports = foldl insert [] + where + insert acc (DocReexport next) = case Array.findIndex (\(DocReexport re) -> re.moduleName == next.moduleName) acc of + Nothing -> Array.snoc acc (DocReexport next) + Just index -> fromMaybe acc $ Array.modifyAt index + (\(DocReexport re) -> DocReexport re { declarations = foldl appendUnique re.declarations next.declarations }) + acc + + appendUnique declarations declaration + | Array.any (sameDeclaration declaration) declarations = declarations + | otherwise = Array.snoc declarations declaration + + sameDeclaration a b = declarationKey a == declarationKey b + + declarationKey (DocDeclaration { info }) = case info of + DeclValue { name } -> Tuple "value" (qualifiedValueName name) + DeclData { name } -> Tuple "type" (qualifiedTypeName name) + DeclForeignData { name } -> Tuple "type" (qualifiedTypeName name) + DeclType { name } -> Tuple "type" (qualifiedTypeName name) + DeclTypeClass { name } -> Tuple "type" (qualifiedTypeName name) + DeclInfix { name, alias: AliasType _ } -> Tuple "type" (qualifiedOperatorName name) + DeclInfix { name } -> Tuple "value" (qualifiedOperatorName name) + + qualifiedValueName (Qualified { moduleName, name }) = (coerce moduleName :: String) <> "." <> (coerce name :: String) + qualifiedTypeName (Qualified { moduleName, name }) = (coerce moduleName :: String) <> "." <> (coerce name :: String) + qualifiedOperatorName (Qualified { moduleName, name }) = (coerce moduleName :: String) <> "." <> (coerce name :: String) + +reexportsOf :: CST.ModuleHeader Void -> Map ModuleName (RefSet ModuleMemberSet) +reexportsOf sourceModule@(CST.ModuleHeader { exports }) = + un SemigroupMap $ fold $ Map.filterWithKey (\k _ -> Set.member k moduleKeys) importSet + where + SemigroupMap importSet = importSetOf sourceModule + moduleKeys = foldMap (foldMap moduleKeyOf <<< delimitedNonEmptyToArray) exports + moduleKeyOf = case _ of + CST.ExportModule _ (CST.Name { name }) -> + Set.singleton (coerce name) + _ -> + mempty + +importSetOf :: CST.ModuleHeader Void -> SemigroupMap ModuleName ImportModuleSet +importSetOf (CST.ModuleHeader { imports }) = + foldMap fromImportDecl imports + where + ModuleMemberSet initial = mempty + + fromImportDecl (CST.ImportDecl { module: CST.Name { name: importModule }, names, qualified }) = + SemigroupMap + $ Map.singleton qualifiedModule + $ SemigroupMap + $ Map.singleton (coerce importModule) memberSet + where + qualifiedModule = case qualified of + Just (Tuple _ (CST.Name { name })) -> + coerce name + Nothing -> + coerce importModule + + memberSet = case names of + Just (Tuple hiding importList) -> case hiding of + Just _ -> RefHiding members + Nothing -> RefSet members + where + members = foldMap fromImport $ delimitedNonEmptyToArray importList + Nothing -> + RefAll + + fromImport = ModuleMemberSet <<< case _ of + CST.ImportValue (CST.Name { name }) -> + initial { values = Set.singleton (coerce name) } + CST.ImportOp (CST.Name { name }) -> + initial { operators = Set.singleton (coerce name) } + CST.ImportType (CST.Name { name }) members -> + initial { types = Map.singleton (coerce name) $ fromDataMembers members } + CST.ImportTypeOp _ (CST.Name { name }) -> + initial { typeOperators = Set.singleton (coerce name) } + CST.ImportClass _ (CST.Name { name }) -> + initial { typeClasses = Set.singleton (coerce name) } + CST.ImportError _ -> + initial + + fromDataMembers = case _ of + Just (CST.DataAll _) -> + RefAll + Just (CST.DataEnumerated names) -> + RefSet + $ Set.fromFoldable + $ map (\(CST.Name { name }) -> coerce name) + $ delimitedToArray names + Nothing -> + RefSet mempty + +delimitedNonEmptyToArray :: forall a. CST.DelimitedNonEmpty a -> Array a +delimitedNonEmptyToArray (CST.Wrapped { value }) = separatedToArray value + +delimitedToArray :: forall a. CST.Delimited a -> Array a +delimitedToArray (CST.Wrapped { value }) = foldMap separatedToArray value + +separatedToArray :: forall a. CST.Separated a -> Array a +separatedToArray (CST.Separated { head, tail }) = Array.cons head $ snd <$> tail diff --git a/docgen/test/Test/Registry/Docgen.purs b/docgen/test/Test/Registry/Docgen.purs new file mode 100644 index 000000000..dc4d5ee30 --- /dev/null +++ b/docgen/test/Test/Registry/Docgen.purs @@ -0,0 +1,495 @@ +module Test.Registry.Docgen (main) where + +import Prelude + +import Control.Monad.Except (runExcept) +import Data.Array as Array +import Data.Array.NonEmpty as NonEmptyArray +import Data.Codec as Codec +import Data.Either (Either(..)) +import Data.Map as Map +import Data.Maybe (Maybe(..)) +import Data.Newtype (unwrap) +import Data.String as String +import Effect (Effect) +import Effect.Aff (Aff) +import JSON as JSON +import JSON.Object as JObject +import Node.Encoding (Encoding(..)) +import Node.FS.Aff as FS.Aff +import PureScript.CST (PartialModule(..), RecoveredParserResult(..), parsePartialModule) +import PureScript.CST.Types as CST +import Registry.Docgen.Codec as Docgen.Codec +import Registry.Docgen.Commonmark (renderMarkdownHTML) +import Registry.Docgen.Convert as Convert +import Registry.Docgen.Docs (DataConstructorName(..), DocChildDeclaration(..), DocChildDeclarationInfo(..), DocConstraint(..), DocDeclaration(..), DocDeclarationInfo(..), DocModule(..), DocPackage(..), DocReexport(..), DocType(..), ForallBinding(..), Ident(..), ModuleName(..), Qualified(..), RawRange(..), Readme(..), SourceArtifact(..), SourcePos(..), SourceSpan(..), TypeName(..), TypeVar(..), ValueName(..), schemaVersion) +import Registry.Docgen.Generate (GenerationError(..)) +import Registry.Docgen.Generate as Generate +import Registry.Docgen.HTML as H +import Registry.Docgen.Legacy.Docs as L +import Registry.Docgen.Legacy.JSON as Legacy.JSON +import Registry.Docgen.Package.Render (defaultPackageLinker, htmlCodeRenderer, renderDeclarationInfo) +import Registry.Docgen.Reexports (ReexportError(..), modulesWithReexports, printReexportError) +import Registry.License as License +import Registry.Location (Location(..)) +import Registry.PackageName (PackageName) +import Registry.PackageName as PackageName +import Registry.Sha256 as Sha256 +import Registry.Test.Assert as Assert +import Registry.Test.Utils as Utils +import Registry.Version (Version) +import Registry.Version as Version +import Test.Spec as Spec +import Test.Spec.Reporter.Console (consoleReporter) +import Test.Spec.Runner.Node (runSpecAndExitProcess) + +main :: Effect Unit +main = runSpecAndExitProcess [ consoleReporter ] do + Spec.describe "registry-docgen" do + Spec.it "round trips schema 1 and rejects unsupported schema versions" do + let encoded = Codec.encode Docgen.Codec.docPackage package + let decoded = runExcept $ Codec.decode Docgen.Codec.docPackage encoded + case decoded of + Left err -> Assert.fail $ show err + Right value -> Codec.encode Docgen.Codec.docPackage value `Assert.shouldEqual` encoded + let unsupported = Utils.fromRight "unsupported JSON" $ JSON.parse $ String.replace (String.Pattern "\"schemaVersion\":1") (String.Replacement "\"schemaVersion\":2") $ JSON.print encoded + case runExcept (Codec.decode Docgen.Codec.docPackage unsupported) of + Left err -> shouldContainString (show err) "Unsupported documentation schema version 2" + Right _ -> Assert.fail "schema version 2 unexpectedly decoded" + let unsupportedOnly = Utils.fromRight "unsupported-only JSON" $ JSON.parse "{\"schemaVersion\":2}" + case runExcept (Codec.decode Docgen.Codec.docPackage unsupportedOnly) of + Left err -> shouldContainString (show err) "Unsupported documentation schema version 2" + Right _ -> Assert.fail "incomplete schema version 2 unexpectedly decoded" + case JSON.toJObject encoded >>= JObject.lookup "dependencies" of + Just dependencies -> case JSON.toJObject dependencies of + Just _ -> pure unit + Nothing -> Assert.fail "Encoded dependencies were not a JSON object" + Nothing -> Assert.fail "Encoded package omitted dependencies" + + Spec.it "round trips and converts visible kind applications" do + let kindApp = TypeKindApp { function: TypeIdent (Ident "f"), arg: TypeIdent (Ident "k") } + let encoded = Codec.encode Docgen.Codec.docType kindApp + case runExcept (Codec.decode Docgen.Codec.docType encoded) of + Right (TypeKindApp { function: TypeIdent (Ident "f"), arg: TypeIdent (Ident "k") }) -> pure unit + _ -> Assert.fail "Canonical kind application did not round trip" + case Convert.fromLegacyModule Map.empty (legacyModuleWithDeclaration "A" (legacyValue "x" (L.KindApp (L.TypeVar "f") (L.TypeVar "k")) Nothing)) of + DocModule { declarations: [ DocDeclaration { info: DeclValue { signature: TypeKindApp { function: TypeIdent (Ident "f"), arg: TypeIdent (Ident "k") } } } ] } -> pure unit + _ -> Assert.fail "Legacy kind application was not preserved" + + Spec.it "converts historical Pursuit JSON using explicit package-relative source paths" do + let json = Utils.fromRight "Failed to parse historical fixture" $ JSON.parse historicalPackage + let legacy = Utils.fromRight "Failed to decode historical fixture" $ Legacy.JSON.decodeDocPackage json + let readme = Readme { content: "# Undefined", extension: Just "md" } + let converted = Utils.fromRight "Failed to convert historical fixture" $ Convert.fromLegacyPackage (Map.singleton (ModuleName "Undefined") "custom/Undefined.purs") sourceArtifactFixture (Just readme) legacy + goldenSource <- FS.Aff.readTextFile UTF8 "docgen/fixtures/undefined-1.0.2.json" + let golden = Utils.fromRight "Failed to parse canonical golden fixture" $ JSON.parse goldenSource + Codec.encode Docgen.Codec.docPackage converted `Assert.shouldEqual` golden + let (DocPackage convertedPackage) = converted + convertedPackage.name `Assert.shouldEqual` packageName "undefined" + convertedPackage.compilerVersion `Assert.shouldEqual` version "0.12.2" + case convertedPackage.modules of + [ DocModule { declarations: [ DocDeclaration { sourceSpan: Just (SourceSpan span) } ] } ] -> + span.path `Assert.shouldEqual` "custom/Undefined.purs" + _ -> + Assert.fail "Expected one converted declaration with a source span" + case Convert.fromLegacyPackage Map.empty sourceArtifactFixture (Just readme) legacy of + Left err -> shouldContainString err "Missing package-relative source path for module Undefined" + Right _ -> Assert.fail "Conversion unexpectedly accepted a missing module source path" + + Spec.it "attributes legacy reexports to their origin module and source path" do + let + legacy = L.DocModule + { comments: Nothing + , declarations: [ legacyValue "own" (L.TypeVar "a") (Just sourceSpan) ] + , name: ModuleName "A" + , reExports: [ L.ReExport { moduleName: L.InPackage { item: ModuleName "B", package: Nothing }, declarations: [ legacyValue "other" (L.TypeVar "b") (Just sourceSpan) ] } ] + } + case Convert.fromLegacyModule (Map.singleton (ModuleName "A") "A.purs") legacy of + DocModule + { declarations: [ DocDeclaration { sourceSpan: Just (SourceSpan { path: "A.purs" }) } ] + , reexports: [ DocReexport { declarations: [ DocDeclaration { info: DeclValue { name: Qualified { moduleName: ModuleName "B" } }, sourceSpan: Nothing } ] } ] + } -> pure unit + _ -> Assert.fail "Legacy reexport inherited owner provenance" + + Spec.describe "canonical package generation" do + Spec.it "is deterministic across input order and records module ownership" do + let generated = Utils.fromRight "Generation failed" $ Generate.generatePackage generationInput + let reversed = Utils.fromRight "Reversed generation failed" $ Generate.generatePackage generationInput { modules = Array.reverse generationInput.modules } + Codec.encode Docgen.Codec.docPackage generated `Assert.shouldEqual` Codec.encode Docgen.Codec.docPackage reversed + case generated of + DocPackage { modules: [ DocModule { name: ModuleName "A", reexports: [ DocReexport { moduleName: ModuleName "B", declarations: [ DocDeclaration { sourceSpan: Just (SourceSpan { path: "src/B.purs" }) } ] } ] } ], resolvedModulePackages } -> do + Map.lookup (ModuleName "A") resolvedModulePackages `Assert.shouldEqual` Just (packageName "example") + Map.lookup (ModuleName "B") resolvedModulePackages `Assert.shouldEqual` Just dependency + _ -> Assert.fail "Generated package did not preserve deterministic ownership, reexports, and source paths" + + Spec.it "rejects duplicate modules and non-package-relative paths" do + let duplicate = generationInput { modules = generationInput.modules <> [ Array.head generationInput.modules # Utils.fromJust "missing module" ] } + case Generate.generatePackage duplicate of + Left (DuplicateModule (ModuleName "B")) -> pure unit + _ -> Assert.fail "Generation unexpectedly accepted a duplicate module" + let absolute = generationInput { modules = map (\moduleInput -> if moduleInput.package == generationInput.name then moduleInput { sourcePath = "/tmp/A.purs" } else moduleInput) generationInput.modules } + case Generate.generatePackage absolute of + Left (InvalidSourcePath (ModuleName "A") "/tmp/A.purs") -> pure unit + _ -> Assert.fail "Generation unexpectedly accepted an absolute source path" + + Spec.describe "re-export resolution" do + Spec.it "resolves direct re-exports without dropping modules" do + let docs = [ emptyModule "A", moduleWithValues "B" [ "foo" ] ] + let resolved = resolveReexports docs [ directReexport "A" "B", plainModule "B" ] + Array.length resolved `Assert.shouldEqual` 2 + reexportSummary "A" resolved `Assert.shouldEqual` [ { moduleName: "B", declarations: 1 } ] + + Spec.it "honors selective imports" do + let docs = [ emptyModule "A", moduleWithValues "B" [ "foo", "bar" ] ] + let resolved = resolveReexports docs [ selectiveReexport "A" "B" "foo", plainModule "B" ] + reexportSummary "A" resolved `Assert.shouldEqual` [ { moduleName: "B", declarations: 1 } ] + + Spec.it "honors hiding imports" do + let docs = [ emptyModule "A", moduleWithValues "B" [ "foo", "bar" ] ] + let resolved = resolveReexports docs [ hidingReexport "A" "B" "foo", plainModule "B" ] + reexportValueNames "A" resolved `Assert.shouldEqual` [ "bar" ] + + Spec.it "preserves constructor restrictions across repeated imports" do + let docs = [ emptyModule "A", moduleWithData "B" ] + let resolved = resolveReexports docs [ partialConstructorReexport, plainModule "B" ] + reexportConstructorNames "A" resolved `Assert.shouldEqual` [ "C1", "C3" ] + + Spec.it "promotes selectively reexported class members with class evidence" do + let docs = [ emptyModule "A", moduleWithClass "B", emptyModule "Prim" ] + let resolved = resolveReexports docs [ selectiveReexport "A" "B" "m", plainModule "B", plainModule "Prim" ] + case reexportDeclarations "A" resolved of + [ DocDeclaration { info: DeclValue { signature: TypeForall { bindings, body: TypeConstrained { constraint: DocConstraint constraint } } } } ] -> do + case NonEmptyArray.toArray bindings, constraint.name, constraint.args of + [ ForallBinding { isVisible: false, name: Ident "a", signature: Just TypeWildcard } ], Qualified { moduleName: ModuleName "B", name: TypeName "C" }, [ TypeIdent (Ident "a") ] -> pure unit + _, _, _ -> Assert.fail "Promoted class evidence had incorrect variables" + _ -> Assert.fail "Class member was not promoted with quantified class evidence" + + Spec.it "resolves module aliases to the owning module" do + let docs = [ emptyModule "A", moduleWithValues "B" [ "foo" ] ] + let resolved = resolveReexports docs [ aliasedReexport "A" "B" "Alias", plainModule "B" ] + reexportSummary "A" resolved `Assert.shouldEqual` [ { moduleName: "B", declarations: 1 } ] + + Spec.it "resolves transitive re-exports to original modules" do + let docs = [ emptyModule "A", emptyModule "B", moduleWithValues "C" [ "foo" ] ] + let resolved = resolveReexports docs [ directReexport "A" "B", directReexport "B" "C", plainModule "C" ] + reexportSummary "A" resolved `Assert.shouldEqual` [ { moduleName: "C", declarations: 1 } ] + reexportSummary "B" resolved `Assert.shouldEqual` [ { moduleName: "C", declarations: 1 } ] + + Spec.it "deduplicates diamond re-exports by origin and namespace" do + let docs = [ emptyModule "A", emptyModule "B", emptyModule "C", moduleWithValues "D" [ "foo" ] ] + let sources = [ twoReexports "A" "B" "C", directReexport "B" "D", directReexport "C" "D", plainModule "D" ] + reexportSummary "A" (resolveReexports docs sources) `Assert.shouldEqual` [ { moduleName: "D", declarations: 1 } ] + + Spec.it "reports missing documentation targets" do + case modulesWithReexports [ emptyModule "A" ] [ parseHeader $ directReexport "A" "Missing" ] of + Left err@(MissingDocsTarget (ModuleName "A") (ModuleName "Missing")) -> + shouldContainString (printReexportError err) "A reexports missing documentation module Missing" + Left err -> Assert.fail $ "Unexpected re-export error: " <> printReexportError err + Right _ -> Assert.fail "Missing documentation target unexpectedly resolved" + + Spec.it "reports docs modules with no source header" do + case modulesWithReexports [ emptyModule "A", emptyModule "B" ] [ parseHeader $ plainModule "A" ] of + Left err@(MissingSourceHeaders names) -> do + names `Assert.shouldEqual` Utils.unsafeNonEmptyArray [ ModuleName "B" ] + shouldContainString (printReexportError err) "Missing parsed source headers" + Left err -> Assert.fail $ "Unexpected re-export error: " <> printReexportError err + Right _ -> Assert.fail "Missing source header unexpectedly resolved" + + Spec.it "reports the complete cycle chain" do + let sources = map parseHeader [ directReexport "A" "B", directReexport "B" "A" ] + case modulesWithReexports [ emptyModule "A", emptyModule "B" ] sources of + Left err@(ReexportCycle names) -> do + names `Assert.shouldEqual` Utils.unsafeNonEmptyArray [ ModuleName "A", ModuleName "B", ModuleName "A" ] + printReexportError err `Assert.shouldEqual` "Reexport cycle: A -> B -> A" + Left err -> Assert.fail $ "Unexpected re-export error: " <> printReexportError err + Right _ -> Assert.fail "Re-export cycle unexpectedly resolved" + + Spec.it "uses dependency package and exact owning version in links" do + let linker = defaultPackageLinker package + linker.getPackageLink dependency `Assert.shouldEqual` + { href: "/packages/dependency/2.3.4", title: "dependency@2.3.4" } + linker.getSourceLink { moduleName: ModuleName "Dependency", sourceSpan } `Assert.shouldEqual` + { href: "https://www.purescript.org/registry-package-viewer/#/dependency/2.3.4/custom/Dependency.purs#4-8" + , title: "dependency@2.3.4/custom/Dependency.purs" + } + + Spec.it "renders foreign data, escapes HTML, and sanitizes unsafe Markdown" do + let code = htmlCodeRenderer (defaultPackageLinker package) (ModuleName "Main") + let rendered = unwrap (renderDeclarationInfo code foreignData).content + shouldContainString rendered "foreign import data" + shouldContainString rendered "Foreign" + unwrap (H.text "") `Assert.shouldEqual` "<unsafe & text>" + let markdown = unwrap (renderMarkdownHTML { safe: true } " [bad](javascript:alert(1)) ****") + String.contains (String.Pattern "