From 9e727f7b6973b63984ddeaf97e42502f0b06d1b1 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Mon, 12 Jan 2026 00:59:44 +0200 Subject: [PATCH 01/20] Integrate registry cron scripts into the job queue --- app-e2e/src/Test/E2E/Endpoint/Scheduler.purs | 148 ++++++++++ app-e2e/src/Test/Main.purs | 5 + app/fixtures/registry-index/tr/an/transferred | 1 + .../registry/metadata/transferred.json | 16 ++ app/src/App/API.purs | 17 +- app/src/App/Main.purs | 43 ++- app/src/App/Server/Scheduler.purs | 268 ++++++++++++++++++ db/schema.sql | 2 + nix/overlay.nix | 8 - nix/test/config.nix | 81 +++++- scripts/src/PackageSetUpdater.purs | 192 ------------- scripts/src/PackageTransferrer.purs | 215 -------------- 12 files changed, 560 insertions(+), 436 deletions(-) create mode 100644 app-e2e/src/Test/E2E/Endpoint/Scheduler.purs create mode 100644 app/fixtures/registry-index/tr/an/transferred create mode 100644 app/fixtures/registry/metadata/transferred.json create mode 100644 app/src/App/Server/Scheduler.purs delete mode 100644 scripts/src/PackageSetUpdater.purs delete mode 100644 scripts/src/PackageTransferrer.purs diff --git a/app-e2e/src/Test/E2E/Endpoint/Scheduler.purs b/app-e2e/src/Test/E2E/Endpoint/Scheduler.purs new file mode 100644 index 00000000..eb35a416 --- /dev/null +++ b/app-e2e/src/Test/E2E/Endpoint/Scheduler.purs @@ -0,0 +1,148 @@ +-- | E2E tests for the Scheduler, covering: +-- | - scheduleLegacyImports: Detects new package versions via GitHub tags +-- | - scheduleTransfers: Detects packages that moved to new GitHub locations +-- | - schedulePackageSetUpdates: Detects recent uploads for package set inclusion +-- | +-- | IMPORTANT: These tests must run BEFORE resetTestState is called, since +-- | the scheduler runs at server startup and creates jobs that would be cleared. +module Test.E2E.Endpoint.Scheduler (spec) where + +import Registry.App.Prelude + +import Data.Array as Array +import Data.Map as Map +import Registry.API.V1 (Job(..)) +import Registry.Location (Location(..)) +import Registry.Operation (AuthenticatedPackageOperation(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Registry.Test.Assert as Assert +import Registry.Version as Version +import Test.E2E.Support.Client as Client +import Test.E2E.Support.Env (E2ESpec) +import Test.Spec as Spec + +spec :: E2ESpec +spec = do + Spec.describe "scheduleLegacyImports" do + Spec.it "enqueues publish jobs for new package versions discovered via GitHub tags" do + -- The scheduler runs at server startup and should have already + -- fetched tags for packages in the registry metadata. + -- prelude has v6.0.1 published but v6.0.2 in tags (per wiremock config) + jobs <- Client.getJobs + + -- Find publish jobs for prelude + let + isPreludePublishJob :: Job -> Boolean + isPreludePublishJob = case _ of + PublishJob { packageName, packageVersion } -> + packageName == unsafeFromRight (PackageName.parse "prelude") + && packageVersion + == unsafeFromRight (Version.parse "6.0.2") + _ -> false + + preludeJob = Array.find isPreludePublishJob jobs + + case preludeJob of + Just (PublishJob { payload }) -> do + -- Verify the compiler is from previous version (prelude@6.0.1 has compilers [0.15.10, 0.15.11]) + -- The scheduler should use the lowest compiler from the previous version for compatibility + let expectedCompiler = unsafeFromRight (Version.parse "0.15.10") + when (payload.compiler /= expectedCompiler) do + Assert.fail $ "Expected compiler 0.15.10 but got " <> Version.print payload.compiler + Just _ -> Assert.fail "Expected PublishJob but got different job type" + Nothing -> do + -- Log what jobs we did find for debugging + let publishJobs = Array.filter isPublishJob jobs + Assert.fail $ "Expected to find a publish job for prelude@6.0.2 but found " + <> show (Array.length publishJobs) + <> " publish jobs: " + <> show (map formatPublishJob publishJobs) + + Spec.it "does not enqueue jobs for already-published versions" do + jobs <- Client.getJobs + + -- prelude v6.0.1 is already published, should NOT have a new job + let + isDuplicateJob :: Job -> Boolean + isDuplicateJob = case _ of + PublishJob { packageName, packageVersion } -> + packageName == unsafeFromRight (PackageName.parse "prelude") + && packageVersion + == unsafeFromRight (Version.parse "6.0.1") + _ -> false + + duplicateJob = Array.find isDuplicateJob jobs + + case duplicateJob of + Nothing -> pure unit -- Good, no duplicate job + Just _ -> Assert.fail "Found unexpected publish job for already-published prelude@6.0.1" + + Spec.describe "scheduleTransfers" do + Spec.it "enqueues transfer jobs when package location changes" do + jobs <- Client.getJobs + let + isTransferredJob :: Job -> Boolean + isTransferredJob = case _ of + TransferJob { packageName } -> + packageName == unsafeFromRight (PackageName.parse "transferred") + _ -> false + case Array.find isTransferredJob jobs of + Just (TransferJob { packageName, payload }) -> do + -- Verify packageName + when (packageName /= unsafeFromRight (PackageName.parse "transferred")) do + Assert.fail $ "Wrong package name: " <> PackageName.print packageName + -- Verify newLocation in payload + case payload.payload of + Transfer { newLocation } -> + case newLocation of + GitHub { owner } -> + when (owner /= "new-owner") do + Assert.fail $ "Expected owner 'new-owner' but got '" <> owner <> "'" + _ -> Assert.fail "Expected GitHub location" + _ -> Assert.fail "Expected Transfer payload" + Just _ -> Assert.fail "Expected TransferJob but got different job type" + Nothing -> do + let transferJobs = Array.filter isTransferJob jobs + Assert.fail $ "Expected to find a transfer job for 'transferred' but found " + <> show (Array.length transferJobs) + <> " transfer jobs" + + Spec.describe "schedulePackageSetUpdates" do + Spec.it "enqueues package set update for recent uploads not in set" do + jobs <- Client.getJobs + let packageSetJobs = Array.filter isPackageSetJob jobs + case Array.head packageSetJobs of + Just (PackageSetJob { payload }) -> + case payload of + Operation.PackageSetUpdate { packages } -> + case Map.lookup (unsafeFromRight $ PackageName.parse "type-equality") packages of + Just (Just _) -> pure unit + _ -> Assert.fail "Expected type-equality in package set update" + Just _ -> Assert.fail "Expected PackageSetJob but got different job type" + Nothing -> Assert.fail "Expected package set job to be enqueued" + +-- | Check if a job is a PublishJob +isPublishJob :: Job -> Boolean +isPublishJob = case _ of + PublishJob _ -> true + _ -> false + +-- | Format a PublishJob for debugging output +formatPublishJob :: Job -> String +formatPublishJob = case _ of + PublishJob { packageName, packageVersion } -> + PackageName.print packageName <> "@" <> Version.print packageVersion + _ -> "" + +-- | Check if a job is a TransferJob +isTransferJob :: Job -> Boolean +isTransferJob = case _ of + TransferJob _ -> true + _ -> false + +-- | Check if a job is a PackageSetJob +isPackageSetJob :: Job -> Boolean +isPackageSetJob = case _ of + PackageSetJob _ -> true + _ -> false diff --git a/app-e2e/src/Test/Main.purs b/app-e2e/src/Test/Main.purs index a5b18d43..310cc27f 100644 --- a/app-e2e/src/Test/Main.purs +++ b/app-e2e/src/Test/Main.purs @@ -6,6 +6,7 @@ import Data.Time.Duration (Milliseconds(..)) import Test.E2E.Endpoint.Jobs as Jobs import Test.E2E.Endpoint.PackageSets as PackageSets import Test.E2E.Endpoint.Publish as Publish +import Test.E2E.Endpoint.Scheduler as Scheduler import Test.E2E.Endpoint.Transfer as Transfer import Test.E2E.Endpoint.Unpublish as Unpublish import Test.E2E.GitHubIssue as GitHubIssue @@ -21,6 +22,10 @@ main :: Effect Unit main = do env <- mkTestEnv runSpecAndExitProcess' config [ consoleReporter ] $ hoistE2E env do + -- The scheduler runs at startup and enqueues a bunch of jobs in the DB, + -- so we need to run these tests without cleaning out the state first + Spec.describe "Scheduler" Scheduler.spec + Spec.before_ resetTestState $ Spec.after_ assertReposClean $ Spec.describe "E2E Tests" do Spec.describe "Endpoints" do Spec.describe "Publish" Publish.spec diff --git a/app/fixtures/registry-index/tr/an/transferred b/app/fixtures/registry-index/tr/an/transferred new file mode 100644 index 00000000..21889f95 --- /dev/null +++ b/app/fixtures/registry-index/tr/an/transferred @@ -0,0 +1 @@ +{"name":"transferred","version":"1.0.0","license":"MIT","location":{"githubOwner":"old-owner","githubRepo":"purescript-transferred"},"ref":"v1.0.0","dependencies":{}} diff --git a/app/fixtures/registry/metadata/transferred.json b/app/fixtures/registry/metadata/transferred.json new file mode 100644 index 00000000..b1972993 --- /dev/null +++ b/app/fixtures/registry/metadata/transferred.json @@ -0,0 +1,16 @@ +{ + "location": { + "githubOwner": "old-owner", + "githubRepo": "purescript-transferred" + }, + "published": { + "1.0.0": { + "bytes": 1000, + "compilers": ["0.15.10"], + "hash": "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + "publishedTime": "2022-01-01T00:00:00.000Z", + "ref": "v1.0.0" + } + }, + "unpublished": {} +} diff --git a/app/src/App/API.purs b/app/src/App/API.purs index 8ebc66ba..8584dc27 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -195,11 +195,18 @@ packageSetUpdate details = do let changeSet = candidates.accepted <#> maybe Remove Update Log.notice "Attempting to build package set update." - PackageSets.upgradeAtomic latestPackageSet (fromMaybe prevCompiler payload.compiler) changeSet >>= case _ of - Left error -> - Except.throw $ "The package set produced from this suggested update does not compile:\n\n" <> error - Right packageSet -> do - let commitMessage = PackageSets.commitMessage latestPackageSet changeSet (un PackageSet packageSet).version + PackageSets.upgradeSequential latestPackageSet (fromMaybe prevCompiler payload.compiler) changeSet >>= case _ of + Nothing -> + Except.throw "No packages could be added to the package set. All packages failed to compile." + Just { failed, succeeded, result: packageSet } -> do + unless (Map.isEmpty failed) do + let + formatFailed = String.joinWith "\n" $ Array.catMaybes $ flip map (Map.toUnfoldable failed) \(Tuple name change) -> + case change of + PackageSets.Update version -> Just $ " - " <> formatPackageVersion name version + PackageSets.Remove -> Nothing + Log.warn $ "Some packages could not be added to the set:\n" <> formatFailed + let commitMessage = PackageSets.commitMessage latestPackageSet succeeded (un PackageSet packageSet).version Registry.writePackageSet packageSet commitMessage Log.notice "Built and released a new package set! Now mirroring to the package-sets repo..." Registry.mirrorPackageSet packageSet diff --git a/app/src/App/Main.purs b/app/src/App/Main.purs index e638cc68..46774a71 100644 --- a/app/src/App/Main.purs +++ b/app/src/App/Main.purs @@ -8,22 +8,33 @@ import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch.Retry as Fetch.Retry import Node.Process as Process -import Registry.App.Server.Env (ServerEnv, createServerEnv) +import Registry.App.Effect.Log as Log +import Registry.App.Effect.Registry as Registry +import Registry.App.Server.Env (createServerEnv, runEffects) import Registry.App.Server.JobExecutor as JobExecutor import Registry.App.Server.Router as Router +import Registry.App.Server.Scheduler as Scheduler main :: Effect Unit -main = do - createServerEnv # Aff.runAff_ case _ of - Left error -> do +main = Aff.launchAff_ do + Aff.attempt createServerEnv >>= case _ of + Left error -> liftEffect do Console.log $ "Failed to start server: " <> Aff.message error Process.exit' 1 Right env -> do - case env.vars.resourceEnv.healthchecksUrl of - Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" - Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl - Aff.launchAff_ $ jobExecutor env - Router.runRouter env + -- Initialize registry repo before launching parallel processes, to avoid + -- race condition where both Scheduler and Job Executor try to clone the + -- Registry at the same time + void $ runEffects env do + Log.info "Initializing registry repo..." + Registry.readAllMetadata + liftEffect do + case env.vars.resourceEnv.healthchecksUrl of + Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" + Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl + Aff.launchAff_ $ withRetryLoop "Scheduler" $ Scheduler.runScheduler env + Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor env + Router.runRouter env where healthcheck :: String -> Aff Unit healthcheck healthchecksUrl = loop limit @@ -63,20 +74,22 @@ main = do Succeeded _ -> do Console.error "Healthchecks returned non-200 status and failure limit reached, will not retry." - jobExecutor :: ServerEnv -> Aff Unit - jobExecutor env = do - loop initialRestartDelay + -- | Run an Aff action in a loop with exponential backoff on failure. + -- | If the action runs for longer than 60 seconds before failing, + -- | the restart delay resets to the initial value (heuristic for stability). + withRetryLoop :: String -> Aff (Either Aff.Error Unit) -> Aff Unit + withRetryLoop name action = loop initialRestartDelay where initialRestartDelay = Milliseconds 100.0 loop restartDelay = do start <- nowUTC - result <- JobExecutor.runJobExecutor env + result <- action end <- nowUTC Console.error case result of - Left error -> "Job executor failed: " <> Aff.message error - Right _ -> "Job executor exited for no reason." + Left error -> name <> " failed: " <> Aff.message error + Right _ -> name <> " exited for no reason." -- This is a heuristic: if the executor keeps crashing immediately, we -- restart with an exponentially increasing delay, but once the executor diff --git a/app/src/App/Server/Scheduler.purs b/app/src/App/Server/Scheduler.purs new file mode 100644 index 00000000..273abb5b --- /dev/null +++ b/app/src/App/Server/Scheduler.purs @@ -0,0 +1,268 @@ +module Registry.App.Server.Scheduler + ( runScheduler + ) where + +import Registry.App.Prelude + +import Data.Array as Array +import Data.Array.NonEmpty as NonEmptyArray +import Data.DateTime as DateTime +import Data.Map as Map +import Data.Set as Set +import Data.String as String +import Data.Time.Duration (Hours(..)) +import Effect.Aff (Milliseconds(..)) +import Effect.Aff as Aff +import Registry.App.Auth as Auth +import Registry.App.CLI.PursVersions as PursVersions +import Registry.App.Effect.Db as Db +import Registry.App.Effect.Env as Env +import Registry.App.Effect.GitHub as GitHub +import Registry.App.Effect.Log as Log +import Registry.App.Effect.PackageSets as PackageSets +import Registry.App.Effect.Registry as Registry +import Registry.App.Legacy.LenientVersion as LenientVersion +import Registry.App.Server.Env (ServerEffects, ServerEnv, runEffects) +import Registry.Foreign.Octokit as Octokit +import Registry.Location (Location(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Registry.PackageSet (PackageSet(..)) +import Run (Run) + +-- | The scheduler loop runs immediately, then every 24 hours. +-- | It checks for work that needs to be enqueued (transfers, package set +-- | updates, legacy imports) and creates the appropriate jobs. +runScheduler :: ServerEnv -> Aff (Either Aff.Error Unit) +runScheduler env = runEffects env do + Log.info "Starting Scheduler" + loop + where + sleepTime = Milliseconds (1000.0 * 60.0 * 60.0 * 12.0) + + loop = do + -- Run all scheduling checks + scheduleTransfers + schedulePackageSetUpdates + scheduleLegacyImports + Log.info "Scheduler cycle complete, sleeping for 24 hours..." + -- Sleep for a while, then run again + liftAff $ Aff.delay sleepTime + loop + +-- | Check for packages that have moved and enqueue transfer jobs. +scheduleTransfers :: Run ServerEffects Unit +scheduleTransfers = do + Log.info "Scheduler: checking for package transfers..." + allMetadata <- Registry.readAllMetadata + + -- Check each package for location changes + transfersNeeded <- Array.catMaybes <$> for (Map.toUnfoldable allMetadata) \(Tuple name (Metadata metadata)) -> + case metadata.location of + Git _ -> pure Nothing -- Skip non-GitHub packages + GitHub registered -> do + -- Fetch tags to see if repo has moved + GitHub.listTags { owner: registered.owner, repo: registered.repo } >>= case _ of + Left _ -> pure Nothing -- Can't fetch tags, skip + Right tags | Array.null tags -> pure Nothing -- No tags, skip + Right tags -> case Array.head tags of + Nothing -> pure Nothing + Just tag -> + -- Parse the tag URL to get actual current location + case tagUrlToRepoUrl tag.url of + Nothing -> pure Nothing + Just actual + | locationsMatch registered actual -> pure Nothing -- No change + | otherwise -> pure $ Just { name, newLocation: GitHub { owner: actual.owner, repo: actual.repo, subdir: registered.subdir } } + + case Array.length transfersNeeded of + 0 -> Log.info "No packages require transferring." + n -> do + Log.info $ show n <> " packages need transferring" + for_ transfersNeeded \{ name, newLocation } -> + enqueueTransferJob name newLocation + +-- | Parse GitHub API tag URL to extract owner/repo +-- | Example: https://api.github.com/repos/octocat/Hello-World/commits/abc123 +tagUrlToRepoUrl :: String -> Maybe { owner :: String, repo :: String } +tagUrlToRepoUrl url = do + noPrefix <- String.stripPrefix (String.Pattern "https://api.github.com/repos/") url + case Array.take 2 $ String.split (String.Pattern "/") noPrefix of + [ owner, repo ] -> Just { owner, repo: String.toLower repo } + _ -> Nothing + +-- | Case-insensitive comparison of GitHub locations +locationsMatch :: forall r. { owner :: String, repo :: String | r } -> { owner :: String, repo :: String } -> Boolean +locationsMatch loc1 loc2 = + String.toLower loc1.owner == String.toLower loc2.owner + && String.toLower loc1.repo + == String.toLower loc2.repo + +enqueueTransferJob :: PackageName -> Location -> Run ServerEffects Unit +enqueueTransferJob name newLocation = do + -- Check if transfer job already exists + existingJob <- Db.selectTransferJob name + case existingJob of + Just _ -> Log.debug $ "Transfer job already exists for " <> PackageName.print name + Nothing -> do + let payload = { name, newLocation } + let rawPayload = stringifyJson Operation.transferCodec payload + { privateKey } <- Env.askPacchettiBotti + case Auth.signPayload { privateKey, rawPayload } of + Left _ -> Log.error $ "Failed to sign transfer for " <> PackageName.print name + Right signature -> do + jobId <- Db.insertTransferJob { payload, rawPayload, signature } + Log.info $ "Enqueued transfer job " <> unwrap jobId <> " for " <> PackageName.print name + +-- | Check for recent uploads and enqueue package set update job. +schedulePackageSetUpdates :: Run ServerEffects Unit +schedulePackageSetUpdates = do + Log.info "Scheduler: checking for package set updates..." + + -- Get the current package set + latestPackageSet <- Registry.readLatestPackageSet >>= case _ of + Nothing -> do + Log.warn "No package set found, skipping package set updates" + pure Nothing + Just set -> pure (Just set) + + for_ latestPackageSet \packageSet -> do + let currentPackages = (un PackageSet packageSet).packages + + -- Find packages uploaded in the last 24 hours that aren't already in the set + recentUploads <- findRecentUploads (Hours 24.0) + let + -- Filter out packages already in the set at the same or newer version + newOrUpdated = recentUploads # Map.filterWithKey \name version -> + case Map.lookup name currentPackages of + -- new package goes in + Nothing -> true + -- as do existing packages with a newer version + Just currentVersion -> version > currentVersion + + if Map.isEmpty newOrUpdated then + Log.info "No new packages for package set update." + else do + Log.info $ "Found " <> show (Map.size newOrUpdated) <> " candidates to validate" + + -- Pre-validate candidates to filter out packages with missing dependencies + manifestIndex <- Registry.readAllManifests + let candidates = PackageSets.validatePackageSetCandidates manifestIndex packageSet (map Just newOrUpdated) + + unless (Map.isEmpty candidates.rejected) do + Log.info $ "Some packages are not eligible for the package set:\n" <> PackageSets.printRejections candidates.rejected + + -- Only enqueue accepted packages (filter out removals, keep only updates) + let accepted = Map.catMaybes candidates.accepted + + if Map.isEmpty accepted then + Log.info "No packages passed validation for package set update." + else do + Log.info $ "Validated " <> show (Map.size accepted) <> " packages for package set update" + + -- Create a package set update payload with only validated packages + let + payload = Operation.PackageSetUpdate + { compiler: Nothing -- Use current compiler + , packages: map Just accepted -- Just version = add/update + } + rawPayload = stringifyJson Operation.packageSetOperationCodec payload + + -- Check if a similar job already exists + existingJob <- Db.selectPackageSetJobByPayload payload + case existingJob of + Just _ -> Log.debug "Package set job with same payload already exists" + Nothing -> do + -- No signature needed for package additions (only for compiler upgrades) + jobId <- Db.insertPackageSetJob { payload, rawPayload, signature: Nothing } + Log.info $ "Enqueued package set job " <> unwrap jobId + +-- | Find the latest version of each package uploaded within the time limit +findRecentUploads :: Hours -> Run ServerEffects (Map PackageName Version) +findRecentUploads limit = do + allMetadata <- Registry.readAllMetadata + now <- nowUTC + + let + getLatestRecentVersion :: Metadata -> Maybe Version + getLatestRecentVersion (Metadata metadata) = + let + recentVersions = Array.catMaybes $ flip map (Map.toUnfoldable metadata.published) \(Tuple version { publishedTime }) -> + let + diff = DateTime.diff now publishedTime + in + if diff <= limit then Just version else Nothing + in + Array.last $ Array.sort recentVersions + + pure $ Map.fromFoldable $ Array.catMaybes $ flip map (Map.toUnfoldable allMetadata) \(Tuple name metadata) -> + map (Tuple name) $ getLatestRecentVersion metadata + +-- | Check for new tags on existing packages and enqueue publish jobs for +-- | versions not yet published. This allows the registry to automatically +-- | import new versions of packages that only have legacy manifests. +scheduleLegacyImports :: Run ServerEffects Unit +scheduleLegacyImports = do + Log.info "Scheduler: checking for new package versions..." + + allMetadata <- Registry.readAllMetadata + let packages = Map.toUnfoldable allMetadata :: Array (Tuple PackageName Metadata) + + for_ packages \(Tuple name (Metadata metadata)) -> do + case metadata.location of + Git _ -> pure unit -- Skip non-GitHub packages for now + GitHub { owner, repo } -> do + GitHub.listTags { owner, repo } >>= case _ of + Left err -> do + Log.debug $ "Failed to fetch tags for " <> PackageName.print name <> ": " <> Octokit.printGitHubError err + Right tags -> do + let + -- Combine published and unpublished versions into a set + publishedVersions = Set.fromFoldable + $ Map.keys metadata.published + <> Map.keys metadata.unpublished + + -- Parse tags as versions and filter out already published ones + newVersions = Array.catMaybes $ tags <#> \tag -> + case LenientVersion.parse tag.name of + Left _ -> Nothing -- Not a valid version tag + Right result -> + let + version = LenientVersion.version result + in + if Set.member version publishedVersions then Nothing -- Already published + else Just { version, ref: tag.name } + + for_ newVersions \{ version, ref } -> + enqueuePublishJob name (Metadata metadata) version ref + + -- Delay between packages to spread GitHub API load + liftAff $ Aff.delay (Milliseconds 500.0) + +-- | Enqueue a publish job for a new package version discovered by the scheduler. +-- | Uses the lowest compiler from the previous published version for compatibility, +-- | falling back to the latest compiler if no previous version exists. +enqueuePublishJob :: PackageName -> Metadata -> Version -> String -> Run ServerEffects Unit +enqueuePublishJob name (Metadata metadata) version ref = do + -- Check if a publish job already exists for this package version + existingJob <- Db.selectPublishJob name version + case existingJob of + Just _ -> Log.debug $ "Publish job already exists for " <> formatPackageVersion name version + Nothing -> do + -- Use the lowest compiler from previous version for compatibility, + -- falling back to latest if no previous version exists + compiler <- case Map.findMax metadata.published of + Just { value: publishedInfo } -> + pure $ NonEmptyArray.head publishedInfo.compilers + Nothing -> NonEmptyArray.last <$> PursVersions.pursVersions + let + payload = + { name + , location: Just metadata.location + , ref + , version + , compiler + , resolutions: Nothing + } + jobId <- Db.insertPublishJob { payload } + Log.info $ "Enqueued legacy publish job " <> unwrap jobId <> " for " <> formatPackageVersion name version diff --git a/db/schema.sql b/db/schema.sql index 65319293..17e06c8d 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -40,6 +40,8 @@ CREATE TABLE matrix_jobs ( CREATE TABLE package_set_jobs ( jobId TEXT PRIMARY KEY NOT NULL, payload JSON NOT NULL, + rawPayload TEXT NOT NULL, + signature TEXT, FOREIGN KEY (jobId) REFERENCES job_info (jobId) ON DELETE CASCADE ); CREATE TABLE logs ( diff --git a/nix/overlay.nix b/nix/overlay.nix index 8ec743a3..8858c140 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -62,14 +62,6 @@ let module = "Registry.Scripts.PackageDeleter"; description = "Delete packages from the registry"; }; - package-set-updater = { - module = "Registry.Scripts.PackageSetUpdater"; - description = "Update package sets"; - }; - package-transferrer = { - module = "Registry.Scripts.PackageTransferrer"; - description = "Transfer packages between storage backends"; - }; solver = { module = "Registry.Scripts.Solver"; description = "Run dependency solver against registry manifests"; diff --git a/nix/test/config.nix b/nix/test/config.nix index 07917444..a23de83c 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -205,6 +205,76 @@ let }; }; } + # Tags for prelude package (used by Scheduler tests) + # Includes v6.0.1 (already published) and v6.0.2 (new version for scheduler to discover) + { + request = { + method = "GET"; + url = "/repos/purescript/purescript-prelude/tags"; + }; + response = { + status = 200; + headers."Content-Type" = "application/json"; + jsonBody = [ + { + name = "v6.0.1"; + commit = { + sha = "abc123def456"; + url = "https://api.github.com/repos/purescript/purescript-prelude/commits/abc123def456"; + }; + } + { + name = "v6.0.2"; + commit = { + sha = "def456abc789"; + url = "https://api.github.com/repos/purescript/purescript-prelude/commits/def456abc789"; + }; + } + ]; + }; + } + # Tags for type-equality package + { + request = { + method = "GET"; + url = "/repos/purescript/purescript-type-equality/tags"; + }; + response = { + status = 200; + headers."Content-Type" = "application/json"; + jsonBody = [ + { + name = "v4.0.1"; + commit = { + sha = "type-eq-sha-401"; + url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-401"; + }; + } + ]; + }; + } + # Tags for transferred package (scheduler transfer detection test) + # Metadata says old-owner but tags point to new-owner - triggers transfer detection + { + request = { + method = "GET"; + url = "/repos/old-owner/purescript-transferred/tags"; + }; + response = { + status = 200; + headers."Content-Type" = "application/json"; + jsonBody = [ + { + name = "v1.0.0"; + commit = { + sha = "transferred-sha-100"; + # Points to NEW owner - scheduler should detect this transfer + url = "https://api.github.com/repos/new-owner/purescript-transferred/commits/transferred-sha-100"; + }; + } + ]; + }; + } # Accept issue comment creation (used by GitHubIssue workflow) { request = { @@ -781,7 +851,7 @@ let # Script to set up git fixtures setupGitFixtures = pkgs.writeShellApplication { name = "setup-git-fixtures"; - runtimeInputs = [ pkgs.git ]; + runtimeInputs = [ pkgs.git pkgs.jq ]; text = '' FIXTURES_DIR="''${1:-${stateDir}/repo-fixtures}" @@ -802,6 +872,15 @@ let cp -r ${rootPath}/app/fixtures/github-packages/console-6.1.0 "$FIXTURES_DIR/purescript/purescript-console" chmod -R u+w "$FIXTURES_DIR/purescript" + # Set type-equality publishedTime to current time for package set update test + # This makes type-equality appear as a "recent upload" so the scheduler will + # detect it and enqueue a package set update job + current_time=$(date -u +"%Y-%m-%dT%H:%M:%S.000Z") + jq --arg time "$current_time" \ + '.published["4.0.1"].publishedTime = $time' \ + "$FIXTURES_DIR/purescript/registry/metadata/type-equality.json" > temp.json && \ + mv temp.json "$FIXTURES_DIR/purescript/registry/metadata/type-equality.json" + for repo in "$FIXTURES_DIR"/purescript/*/; do cd "$repo" git init -b master && git add . diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs deleted file mode 100644 index 29423cf7..00000000 --- a/scripts/src/PackageSetUpdater.purs +++ /dev/null @@ -1,192 +0,0 @@ -module Registry.Scripts.PackageSetUpdater where - -import Registry.App.Prelude - -import ArgParse.Basic (ArgParser) -import ArgParse.Basic as Arg -import Data.Array as Array -import Data.Array.NonEmpty as NonEmptyArray -import Data.DateTime as DateTime -import Data.FoldableWithIndex (foldMapWithIndex) -import Data.Formatter.DateTime as Formatter.DateTime -import Data.Map as Map -import Data.Number.Format as Number.Format -import Data.String as String -import Data.Time.Duration (Hours(..)) -import Effect.Aff as Aff -import Effect.Class.Console as Console -import Node.Path as Path -import Node.Process as Process -import Registry.App.CLI.Git as Git -import Registry.App.Effect.Cache as Cache -import Registry.App.Effect.Env as Env -import Registry.App.Effect.GitHub as GitHub -import Registry.App.Effect.Log (LOG) -import Registry.App.Effect.Log as Log -import Registry.App.Effect.PackageSets (Change(..), PACKAGE_SETS) -import Registry.App.Effect.PackageSets as PackageSets -import Registry.App.Effect.Registry (REGISTRY) -import Registry.App.Effect.Registry as Registry -import Registry.App.Effect.Storage as Storage -import Registry.Foreign.FSExtra as FS.Extra -import Registry.Foreign.Octokit as Octokit -import Registry.Internal.Format as Internal.Format -import Registry.Version as Version -import Run (AFF, EFFECT, Run) -import Run as Run -import Run.Except (EXCEPT) -import Run.Except as Except - -data PublishMode = GeneratePackageSet | CommitPackageSet - -derive instance Eq PublishMode - -parser :: ArgParser PublishMode -parser = Arg.choose "command" - [ Arg.flag [ "generate" ] - "Generate a new package set without committing the results." - $> GeneratePackageSet - , Arg.flag [ "commit" ] - "Generate a new package set and commit the results." - $> CommitPackageSet - ] - -main :: Effect Unit -main = Aff.launchAff_ do - args <- Array.drop 2 <$> liftEffect Process.argv - let description = "A script for updating the package sets." - mode <- case Arg.parseArgs "package-set-updater" description parser args of - Left err -> Console.log (Arg.printArgError err) *> liftEffect (Process.exit' 1) - Right command -> pure command - - -- Environment - _ <- Env.loadEnvFile ".env" - - { token, write } <- case mode of - GeneratePackageSet -> do - Env.lookupOptional Env.githubToken >>= case _ of - Nothing -> do - token <- Env.lookupRequired Env.pacchettibottiToken - pure { token, write: Registry.ReadOnly } - Just token -> - pure { token, write: Registry.ReadOnly } - CommitPackageSet -> do - token <- Env.lookupRequired Env.pacchettibottiToken - pure { token, write: Registry.CommitAs (Git.pacchettibottiCommitter token) } - - -- Package sets - let packageSetsEnv = { workdir: Path.concat [ scratchDir, "package-set-build" ] } - - -- GitHub - resourceEnv <- Env.lookupResourceEnv - octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl - - -- Caching - let cache = Path.concat [ scratchDir, ".cache" ] - FS.Extra.ensureDirectory cache - githubCacheRef <- Cache.newCacheRef - registryCacheRef <- Cache.newCacheRef - - -- Registry - debouncer <- Registry.newDebouncer - let - registryEnv :: Registry.RegistryEnv - registryEnv = - { write - , pull: Git.ForceClean - , repos: Registry.defaultRepos - , workdir: scratchDir - , debouncer - , cacheRef: registryCacheRef - } - - -- Logging - now <- nowUTC - let logDir = Path.concat [ scratchDir, "logs" ] - FS.Extra.ensureDirectory logDir - let logFile = "package-set-updater-" <> String.take 19 (Formatter.DateTime.format Internal.Format.iso8601DateTime now) <> ".log" - let logPath = Path.concat [ logDir, logFile ] - - updater - # PackageSets.interpret (PackageSets.handle packageSetsEnv) - # Registry.interpret (Registry.handle registryEnv) - # Storage.interpret (Storage.handleReadOnly cache) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - # Except.catch (\msg -> Log.error msg *> Run.liftEffect (Process.exit' 1)) - # Log.interpret (\log -> Log.handleTerminal Normal log *> Log.handleFs Verbose logPath log) - # Env.runResourceEnv resourceEnv - # Run.runBaseAff' - -updater :: forall r. Run (REGISTRY + PACKAGE_SETS + LOG + EXCEPT String + AFF + EFFECT + r) Unit -updater = do - prevPackageSet <- Registry.readLatestPackageSet >>= case _ of - Nothing -> Except.throw "No previous package set found, cannot continue." - Just set -> pure set - - PackageSets.validatePackageSet prevPackageSet - - let compiler = (un PackageSet prevPackageSet).compiler - - Log.info $ "Using compiler " <> Version.print compiler - - let uploadHours = 24.0 - recentUploads <- findRecentUploads (Hours uploadHours) - - manifestIndex <- Registry.readAllManifests - let candidates = PackageSets.validatePackageSetCandidates manifestIndex prevPackageSet (map Just recentUploads.eligible) - unless (Map.isEmpty candidates.rejected) do - Log.info $ "Some packages uploaded in the last " <> Number.Format.toString uploadHours <> " hours are not eligible for the automated package sets." - Log.info $ PackageSets.printRejections candidates.rejected - - if Map.isEmpty candidates.accepted then do - Log.info "No eligible additions, updates, or removals to produce a new package set." - else do - -- You can't remove packages via the automatic updater. - let eligible = Map.catMaybes candidates.accepted - let listPackages = foldMapWithIndex \name version -> [ formatPackageVersion name version ] - Log.info $ "Found package versions eligible for inclusion in package set: " <> Array.foldMap (append "\n - ") (listPackages eligible) - PackageSets.upgradeSequential prevPackageSet compiler (map (maybe Remove Update) candidates.accepted) >>= case _ of - Nothing -> do - Log.info "No packages could be added to the set. All packages failed." - Just { failed, succeeded, result } -> do - let - listChanges = foldMapWithIndex \name -> case _ of - Remove -> [] - Update version -> [ formatPackageVersion name version ] - unless (Map.isEmpty failed) do - Log.info $ "Some packages could not be added to the set: " <> Array.foldMap (append "\n - ") (listChanges failed) - Log.info $ "New packages were added to the set: " <> Array.foldMap (append "\n - ") (listChanges succeeded) - -- We only include the successful changes in the commit message. - let commitMessage = PackageSets.commitMessage prevPackageSet succeeded (un PackageSet result).version - Registry.writePackageSet result commitMessage - Log.info "Built and released a new package set! Now mirroring to the package-sets repo..." - Registry.mirrorPackageSet result - Log.info "Mirrored a new legacy package set." - -type RecentUploads = - { eligible :: Map PackageName Version - , ineligible :: Map PackageName (NonEmptyArray Version) - } - -findRecentUploads :: forall r. Hours -> Run (REGISTRY + EXCEPT String + EFFECT + r) RecentUploads -findRecentUploads limit = do - allMetadata <- Registry.readAllMetadata - now <- nowUTC - - let - uploads = Map.fromFoldable do - Tuple name (Metadata metadata) <- Map.toUnfoldable allMetadata - versions <- Array.fromFoldable $ NonEmptyArray.fromArray do - Tuple version { publishedTime } <- Map.toUnfoldable metadata.published - let diff = DateTime.diff now publishedTime - guard (diff <= limit) - pure version - pure (Tuple name versions) - - deduplicated = uploads # flip foldlWithIndex { ineligible: Map.empty, eligible: Map.empty } \name acc versions -> do - let { init, last } = NonEmptyArray.unsnoc versions - case NonEmptyArray.fromArray init of - Nothing -> acc { eligible = Map.insert name last acc.eligible } - Just entries -> acc { eligible = Map.insert name last acc.eligible, ineligible = Map.insert name entries acc.ineligible } - - pure deduplicated diff --git a/scripts/src/PackageTransferrer.purs b/scripts/src/PackageTransferrer.purs deleted file mode 100644 index 31e85919..00000000 --- a/scripts/src/PackageTransferrer.purs +++ /dev/null @@ -1,215 +0,0 @@ -module Registry.Scripts.PackageTransferrer where - -import Registry.App.Prelude - -import Data.Array as Array -import Data.Codec.JSON as CJ -import Data.Codec.JSON.Common as CJ.Common -import Data.Codec.JSON.Record as CJ.Record -import Data.Formatter.DateTime as Formatter.DateTime -import Data.Map as Map -import Data.String as String -import Effect.Ref as Ref -import Node.Path as Path -import Node.Process as Process -import Registry.App.API as API -import Registry.App.Auth as Auth -import Registry.App.CLI.Git as Git -import Registry.App.Effect.Cache as Cache -import Registry.App.Effect.Env as Env -import Registry.App.Effect.GitHub (GITHUB) -import Registry.App.Effect.GitHub as GitHub -import Registry.App.Effect.Log (LOG) -import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry (REGISTRY) -import Registry.App.Effect.Registry as Registry -import Registry.App.Effect.Storage as Storage -import Registry.App.Legacy.LenientVersion as LenientVersion -import Registry.App.Legacy.Types (RawPackageName(..)) -import Registry.Foreign.FSExtra as FS.Extra -import Registry.Foreign.Octokit (Tag) -import Registry.Foreign.Octokit as Octokit -import Registry.Internal.Format as Internal.Format -import Registry.Location as Location -import Registry.Operation (AuthenticatedPackageOperation(..)) -import Registry.Operation as Operation -import Registry.Operation.Validation as Operation.Validation -import Registry.PackageName as PackageName -import Registry.Scripts.LegacyImporter as LegacyImporter -import Run (Run) -import Run as Run -import Run.Except (EXCEPT) -import Run.Except as Except -import Run.Except as Run.Except - -main :: Effect Unit -main = launchAff_ do - - -- Environment - _ <- Env.loadEnvFile ".env" - token <- Env.lookupRequired Env.pacchettibottiToken - publicKey <- Env.lookupRequired Env.pacchettibottiED25519Pub - privateKey <- Env.lookupRequired Env.pacchettibottiED25519 - resourceEnv <- Env.lookupResourceEnv - - -- Caching - let cache = Path.concat [ scratchDir, ".cache" ] - FS.Extra.ensureDirectory cache - githubCacheRef <- Cache.newCacheRef - registryCacheRef <- Cache.newCacheRef - - -- GitHub - octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl - - -- Registry - debouncer <- Registry.newDebouncer - let - registryEnv :: Registry.RegistryEnv - registryEnv = - { write: Registry.CommitAs (Git.pacchettibottiCommitter token) - , pull: Git.ForceClean - , repos: Registry.defaultRepos - , workdir: scratchDir - , debouncer - , cacheRef: registryCacheRef - } - - -- Logging - now <- nowUTC - let logDir = Path.concat [ scratchDir, "logs" ] - FS.Extra.ensureDirectory logDir - let logFile = "package-transferrer-" <> String.take 19 (Formatter.DateTime.format Internal.Format.iso8601DateTime now) <> ".log" - let logPath = Path.concat [ logDir, logFile ] - - transfer - # Registry.interpret (Registry.handle registryEnv) - # Storage.interpret (Storage.handleReadOnly cache) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - # Except.catch (\msg -> Log.error msg *> Run.liftEffect (Process.exit' 1)) - # Log.interpret (\log -> Log.handleTerminal Normal log *> Log.handleFs Verbose logPath log) - # Env.runPacchettiBottiEnv { privateKey, publicKey } - # Env.runResourceEnv resourceEnv - # Run.runBaseAff' - -transfer :: forall r. Run (API.AuthenticatedEffects + r) Unit -transfer = do - Log.info "Processing legacy registry..." - allMetadata <- Registry.readAllMetadata - { bower, new } <- Registry.readLegacyRegistry - let packages = Map.union bower new - Log.info "Reading latest locations for legacy registry packages..." - locations <- latestLocations allMetadata packages - let needsTransfer = Map.catMaybes locations - case Map.size needsTransfer of - 0 -> Log.info "No packages require transferring." - n -> do - Log.info $ Array.fold [ show n, " packages need transferring: ", printJson (CJ.Common.strMap packageLocationsCodec) needsTransfer ] - _ <- transferAll packages needsTransfer - Log.info "Completed transfers!" - -transferAll :: forall r. Map String String -> Map String PackageLocations -> Run (API.AuthenticatedEffects + r) (Map String String) -transferAll packages packageLocations = do - packagesRef <- liftEffect (Ref.new packages) - forWithIndex_ packageLocations \package locations -> do - let newPackageLocation = locations.tagLocation - transferPackage package newPackageLocation - let url = locationToPackageUrl newPackageLocation - liftEffect $ Ref.modify_ (Map.insert package url) packagesRef - liftEffect $ Ref.read packagesRef - -transferPackage :: forall r. String -> Location -> Run (API.AuthenticatedEffects + r) Unit -transferPackage rawPackageName newLocation = do - name <- case PackageName.parse (stripPureScriptPrefix rawPackageName) of - Left _ -> Except.throw $ "Could not transfer " <> rawPackageName <> " because it is not a valid package name." - Right value -> pure value - - let - payload = { name, newLocation } - rawPayload = stringifyJson Operation.transferCodec payload - - { privateKey } <- Env.askPacchettiBotti - - signature <- case Auth.signPayload { privateKey, rawPayload } of - Left _ -> Except.throw "Error signing transfer." - Right signature -> pure signature - - API.authenticated - { payload: Transfer payload - , rawPayload - , signature - } - -type PackageLocations = - { registeredLocation :: Location - , tagLocation :: Location - } - -packageLocationsCodec :: CJ.Codec PackageLocations -packageLocationsCodec = CJ.named "PackageLocations" $ CJ.Record.object - { registeredLocation: Location.codec - , tagLocation: Location.codec - } - -latestLocations :: forall r. Map PackageName Metadata -> Map String String -> Run (REGISTRY + GITHUB + LOG + EXCEPT String + r) (Map String (Maybe PackageLocations)) -latestLocations allMetadata packages = forWithIndex packages \package location -> do - let rawName = RawPackageName (stripPureScriptPrefix package) - Run.Except.runExceptAt LegacyImporter._exceptPackage (LegacyImporter.validatePackage rawName location) >>= case _ of - Left { error: LegacyImporter.PackageURLRedirects { received, registered } } -> do - let newLocation = GitHub { owner: received.owner, repo: received.repo, subdir: Nothing } - Log.info $ "Package " <> package <> " has moved to " <> locationToPackageUrl newLocation - if Operation.Validation.locationIsUnique newLocation allMetadata then do - Log.info "New location is unique; package will be transferred." - pure $ Just - { registeredLocation: GitHub { owner: registered.owner, repo: registered.repo, subdir: Nothing } - , tagLocation: newLocation - } - else do - Log.info "Package will not be transferred! New location is already in use." - pure Nothing - Left _ -> pure Nothing - Right packageResult | Array.null packageResult.tags -> pure Nothing - Right packageResult -> do - Registry.readMetadata packageResult.name >>= case _ of - Nothing -> do - Log.error $ "Cannot verify location of " <> PackageName.print packageResult.name <> " because it has no metadata." - pure Nothing - Just metadata -> case latestPackageLocations packageResult metadata of - Left error -> do - Log.warn $ "Could not verify location of " <> PackageName.print packageResult.name <> ": " <> error - pure Nothing - Right locations - | locationsMatch locations.registeredLocation locations.tagLocation -> pure Nothing - | otherwise -> pure $ Just locations - where - -- The eq instance for locations has case sensitivity, but GitHub doesn't care. - locationsMatch :: Location -> Location -> Boolean - locationsMatch (GitHub location1) (GitHub location2) = - (String.toLower location1.repo == String.toLower location2.repo) - && (String.toLower location1.owner == String.toLower location2.owner) - locationsMatch _ _ = - unsafeCrashWith "Only GitHub locations can be considered in legacy registries." - -latestPackageLocations :: LegacyImporter.PackageResult -> Metadata -> Either String PackageLocations -latestPackageLocations package (Metadata { location, published }) = do - let - isMatchingTag :: Version -> Tag -> Boolean - isMatchingTag version tag = fromMaybe false do - tagVersion <- hush $ LenientVersion.parse tag.name - pure $ version == LenientVersion.version tagVersion - - matchingTag <- do - if Map.isEmpty published then do - note "No repo tags exist" $ Array.head package.tags - else do - Tuple version _ <- note "No published versions" $ Array.last (Map.toUnfoldable published) - note "No versions match repo tags" $ Array.find (isMatchingTag version) package.tags - tagUrl <- note ("Could not parse tag url " <> matchingTag.url) $ LegacyImporter.tagUrlToRepoUrl matchingTag.url - let tagLocation = GitHub { owner: tagUrl.owner, repo: tagUrl.repo, subdir: Nothing } - pure { registeredLocation: location, tagLocation } - -locationToPackageUrl :: Location -> String -locationToPackageUrl = case _ of - GitHub { owner, repo } -> - Array.fold [ "https://github.com/", owner, "/", repo, ".git" ] - Git _ -> - unsafeCrashWith "Git urls cannot be registered." From 2206f6d77d413bb833b293f8c07d8b16febe17e6 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Mon, 12 Jan 2026 01:10:36 +0200 Subject: [PATCH 02/20] Format nix files --- nix/test/config.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nix/test/config.nix b/nix/test/config.nix index a23de83c..de5ff3e1 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -851,7 +851,10 @@ let # Script to set up git fixtures setupGitFixtures = pkgs.writeShellApplication { name = "setup-git-fixtures"; - runtimeInputs = [ pkgs.git pkgs.jq ]; + runtimeInputs = [ + pkgs.git + pkgs.jq + ]; text = '' FIXTURES_DIR="''${1:-${stateDir}/repo-fixtures}" From 0aba96c35cf25427cc431f65d392d877b9b02e84 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Thu, 15 Jan 2026 22:09:34 +0200 Subject: [PATCH 03/20] Reinstate matrix jobs test at startup, cleanup other scheduler tests --- .../src/Test/E2E/Endpoint/PackageSets.purs | 8 ++ .../Endpoint/{Scheduler.purs => Startup.purs} | 83 ++++++++++---- app-e2e/src/Test/E2E/Support/Env.purs | 63 ++++++++++- app-e2e/src/Test/E2E/Support/Fixtures.purs | 28 ++++- app-e2e/src/Test/Main.purs | 11 +- .../unsafe-coerce-6.0.0/bower.json | 21 ++++ .../unsafe-coerce-6.0.0/src/Unsafe/Coerce.js | 5 + .../src/Unsafe/Coerce.purs | 26 +++++ app/fixtures/registry-index/tr/an/transferred | 1 - .../unsafe-coerce-6.0.0.tar.gz | Bin 0 -> 1310 bytes app/fixtures/registry/metadata/prelude.json | 3 +- .../registry/metadata/transferred.json | 16 --- .../registry/metadata/type-equality.json | 2 +- nix/test/config.nix | 107 +++++++++++++----- 14 files changed, 291 insertions(+), 83 deletions(-) rename app-e2e/src/Test/E2E/Endpoint/{Scheduler.purs => Startup.purs} (62%) create mode 100644 app/fixtures/github-packages/unsafe-coerce-6.0.0/bower.json create mode 100644 app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.js create mode 100644 app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.purs delete mode 100644 app/fixtures/registry-index/tr/an/transferred create mode 100644 app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz delete mode 100644 app/fixtures/registry/metadata/transferred.json diff --git a/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs b/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs index 502853fb..14b85abe 100644 --- a/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs +++ b/app-e2e/src/Test/E2E/Endpoint/PackageSets.purs @@ -16,6 +16,10 @@ spec :: E2ESpec spec = do Spec.describe "Package Sets endpoint" do Spec.it "accepts unauthenticated add/upgrade requests" do + -- First publish unsafe-coerce to create the tarball in storage + { jobId: publishJobId } <- Client.publish Fixtures.unsafeCoercePublishData + _ <- Env.pollJobOrFail publishJobId + -- Now add it to the package set { jobId } <- Client.packageSets Fixtures.packageSetAddRequest job <- Env.pollJobOrFail jobId Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust @@ -47,6 +51,10 @@ spec = do Assert.shouldSatisfy (V1.jobInfo job).finishedAt isJust Spec.it "returns existing job for duplicate requests" do + -- First publish unsafe-coerce so the package set request is valid + { jobId: publishJobId } <- Client.publish Fixtures.unsafeCoercePublishData + _ <- Env.pollJobOrFail publishJobId + -- Now test that duplicate requests return the same job ID { jobId: firstJobId } <- Client.packageSets Fixtures.packageSetAddRequest { jobId: secondJobId } <- Client.packageSets Fixtures.packageSetAddRequest Assert.shouldEqual firstJobId secondJobId diff --git a/app-e2e/src/Test/E2E/Endpoint/Scheduler.purs b/app-e2e/src/Test/E2E/Endpoint/Startup.purs similarity index 62% rename from app-e2e/src/Test/E2E/Endpoint/Scheduler.purs rename to app-e2e/src/Test/E2E/Endpoint/Startup.purs index eb35a416..db804508 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Scheduler.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Startup.purs @@ -1,11 +1,12 @@ --- | E2E tests for the Scheduler, covering: +-- | E2E tests for the Scheduler and JobExecutor startup, covering: -- | - scheduleLegacyImports: Detects new package versions via GitHub tags -- | - scheduleTransfers: Detects packages that moved to new GitHub locations -- | - schedulePackageSetUpdates: Detects recent uploads for package set inclusion +-- | - checkIfNewCompiler: Detects new compiler and enqueues matrix jobs -- | -- | IMPORTANT: These tests must run BEFORE resetTestState is called, since -- | the scheduler runs at server startup and creates jobs that would be cleared. -module Test.E2E.Endpoint.Scheduler (spec) where +module Test.E2E.Endpoint.Startup (spec) where import Registry.App.Prelude @@ -28,24 +29,24 @@ spec = do Spec.it "enqueues publish jobs for new package versions discovered via GitHub tags" do -- The scheduler runs at server startup and should have already -- fetched tags for packages in the registry metadata. - -- prelude has v6.0.1 published but v6.0.2 in tags (per wiremock config) + -- type-equality has v4.0.1 published but v4.0.2 in tags (per wiremock config) jobs <- Client.getJobs - -- Find publish jobs for prelude + -- Find publish jobs for type-equality let - isPreludePublishJob :: Job -> Boolean - isPreludePublishJob = case _ of + isTypeEqualityPublishJob :: Job -> Boolean + isTypeEqualityPublishJob = case _ of PublishJob { packageName, packageVersion } -> - packageName == unsafeFromRight (PackageName.parse "prelude") + packageName == unsafeFromRight (PackageName.parse "type-equality") && packageVersion - == unsafeFromRight (Version.parse "6.0.2") + == unsafeFromRight (Version.parse "4.0.2") _ -> false - preludeJob = Array.find isPreludePublishJob jobs + typeEqualityJob = Array.find isTypeEqualityPublishJob jobs - case preludeJob of + case typeEqualityJob of Just (PublishJob { payload }) -> do - -- Verify the compiler is from previous version (prelude@6.0.1 has compilers [0.15.10, 0.15.11]) + -- Verify the compiler is from previous version (type-equality@4.0.1 has compilers [0.15.10, 0.15.11]) -- The scheduler should use the lowest compiler from the previous version for compatibility let expectedCompiler = unsafeFromRight (Version.parse "0.15.10") when (payload.compiler /= expectedCompiler) do @@ -54,7 +55,7 @@ spec = do Nothing -> do -- Log what jobs we did find for debugging let publishJobs = Array.filter isPublishJob jobs - Assert.fail $ "Expected to find a publish job for prelude@6.0.2 but found " + Assert.fail $ "Expected to find a publish job for type-equality@4.0.2 but found " <> show (Array.length publishJobs) <> " publish jobs: " <> show (map formatPublishJob publishJobs) @@ -62,49 +63,50 @@ spec = do Spec.it "does not enqueue jobs for already-published versions" do jobs <- Client.getJobs - -- prelude v6.0.1 is already published, should NOT have a new job + -- type-equality v4.0.1 is already published, should NOT have a new job let isDuplicateJob :: Job -> Boolean isDuplicateJob = case _ of PublishJob { packageName, packageVersion } -> - packageName == unsafeFromRight (PackageName.parse "prelude") + packageName == unsafeFromRight (PackageName.parse "type-equality") && packageVersion - == unsafeFromRight (Version.parse "6.0.1") + == unsafeFromRight (Version.parse "4.0.1") _ -> false duplicateJob = Array.find isDuplicateJob jobs case duplicateJob of Nothing -> pure unit -- Good, no duplicate job - Just _ -> Assert.fail "Found unexpected publish job for already-published prelude@6.0.1" + Just _ -> Assert.fail "Found unexpected publish job for already-published type-equality@4.0.1" Spec.describe "scheduleTransfers" do Spec.it "enqueues transfer jobs when package location changes" do + -- type-equality metadata says old-owner, but tags point to purescript jobs <- Client.getJobs let - isTransferredJob :: Job -> Boolean - isTransferredJob = case _ of + isTypeEqualityTransferJob :: Job -> Boolean + isTypeEqualityTransferJob = case _ of TransferJob { packageName } -> - packageName == unsafeFromRight (PackageName.parse "transferred") + packageName == unsafeFromRight (PackageName.parse "type-equality") _ -> false - case Array.find isTransferredJob jobs of + case Array.find isTypeEqualityTransferJob jobs of Just (TransferJob { packageName, payload }) -> do -- Verify packageName - when (packageName /= unsafeFromRight (PackageName.parse "transferred")) do + when (packageName /= unsafeFromRight (PackageName.parse "type-equality")) do Assert.fail $ "Wrong package name: " <> PackageName.print packageName -- Verify newLocation in payload case payload.payload of Transfer { newLocation } -> case newLocation of GitHub { owner } -> - when (owner /= "new-owner") do - Assert.fail $ "Expected owner 'new-owner' but got '" <> owner <> "'" + when (owner /= "purescript") do + Assert.fail $ "Expected owner 'purescript' but got '" <> owner <> "'" _ -> Assert.fail "Expected GitHub location" _ -> Assert.fail "Expected Transfer payload" Just _ -> Assert.fail "Expected TransferJob but got different job type" Nothing -> do let transferJobs = Array.filter isTransferJob jobs - Assert.fail $ "Expected to find a transfer job for 'transferred' but found " + Assert.fail $ "Expected to find a transfer job for 'type-equality' but found " <> show (Array.length transferJobs) <> " transfer jobs" @@ -122,6 +124,39 @@ spec = do Just _ -> Assert.fail "Expected PackageSetJob but got different job type" Nothing -> Assert.fail "Expected package set job to be enqueued" + Spec.describe "checkIfNewCompiler" do + Spec.it "enqueues matrix jobs for packages with no dependencies when new compiler detected" do + -- The test env has compilers 0.15.10 and 0.15.11 available. + -- prelude@6.0.1 fixture only has compiler 0.15.10 in metadata. + -- So 0.15.11 should be detected as "new" at startup, triggering + -- matrix jobs for packages with no dependencies. + jobs <- Client.getJobs + let + isNewCompilerMatrixJob :: Job -> Boolean + isNewCompilerMatrixJob = case _ of + MatrixJob { compilerVersion } -> + compilerVersion == unsafeFromRight (Version.parse "0.15.11") + _ -> false + + matrixJobs = Array.filter isNewCompilerMatrixJob jobs + + -- Get package names from matrix jobs + matrixPackages = Array.mapMaybe + ( \j -> case j of + MatrixJob { packageName } -> Just packageName + _ -> Nothing + ) + matrixJobs + + -- Should have matrix jobs for packages with no dependencies + -- prelude has no dependencies, so it should get a matrix job + let preludeName = unsafeFromRight (PackageName.parse "prelude") + unless (Array.elem preludeName matrixPackages) do + Assert.fail $ "Expected matrix job for prelude with compiler 0.15.11, found: " + <> show (Array.length matrixJobs) + <> " matrix jobs for packages: " + <> show (map PackageName.print matrixPackages) + -- | Check if a job is a PublishJob isPublishJob :: Job -> Boolean isPublishJob = case _ of diff --git a/app-e2e/src/Test/E2E/Support/Env.purs b/app-e2e/src/Test/E2E/Support/Env.purs index 06c8d47b..ca988893 100644 --- a/app-e2e/src/Test/E2E/Support/Env.purs +++ b/app-e2e/src/Test/E2E/Support/Env.purs @@ -16,6 +16,7 @@ module Test.E2E.Support.Env , resetTestState , resetDatabase , resetGitFixtures + , stashGitFixtures , resetLogs , resetGitHubRequestCache , pollJobOrFail @@ -25,6 +26,7 @@ module Test.E2E.Support.Env , gitStatus , isCleanGitStatus , waitForAllMatrixJobs + , waitForAllPendingJobs , isMatrixJobFor , readMetadata , readManifestIndexEntry @@ -99,6 +101,10 @@ runE2E env = flip runReaderT env -- | Resets: database, git fixtures, storage mock, and logs. resetTestState :: E2E Unit resetTestState = do + -- Wait for any pending jobs to complete before clearing state. + -- This is important because startup jobs (like matrix jobs from new compiler + -- detection) may still be running when this is called. + waitForAllPendingJobs resetDatabase resetGitFixtures WireMock.clearStorageRequests @@ -124,9 +130,10 @@ resetDatabase = do -- | Reset the git fixtures to restore original state. -- | This restores metadata files modified by unpublish/transfer operations. -- | --- | Strategy: Reset the origin repos to their initial-fixture tag (created during --- | setup), then delete the server's scratch git clones. The server will --- | re-clone fresh copies on the next operation, ensuring a clean cache state. +-- | Strategy: Reset the origin repos to the `post-startup` tag if it exists (created +-- | by stashGitFixtures after startup jobs complete), otherwise fall back to the +-- | `initial-fixture` tag. Then delete the server's scratch git clones so the +-- | server will re-clone fresh copies on the next operation. resetGitFixtures :: E2E Unit resetGitFixtures = do { stateDir } <- ask @@ -140,13 +147,41 @@ resetGitFixtures = do deleteGitClones scratchDir where resetOrigin dir = do - void $ gitOrFail [ "reset", "--hard", "initial-fixture" ] dir + -- Try to reset to post-startup tag first, fall back to initial-fixture + tag <- hasTag "post-startup" dir + let targetTag = if tag then "post-startup" else "initial-fixture" + void $ gitOrFail [ "reset", "--hard", targetTag ] dir void $ gitOrFail [ "clean", "-fd" ] dir + hasTag tagName dir = do + result <- liftAff $ Git.gitCLI [ "tag", "-l", tagName ] (Just dir) + pure $ case result of + Right output -> String.contains (String.Pattern tagName) output + Left _ -> false + deleteGitClones scratchDir = do liftAff $ FS.Extra.remove $ Path.concat [ scratchDir, "registry" ] liftAff $ FS.Extra.remove $ Path.concat [ scratchDir, "registry-index" ] +-- | Stash the current git fixtures state by creating a `post-startup` tag. +-- | This should be called after startup jobs (like matrix jobs from new compiler +-- | detection) have completed, so that resetGitFixtures can restore to this +-- | state instead of the initial fixtures. +stashGitFixtures :: E2E Unit +stashGitFixtures = do + fixturesDir <- liftEffect $ Env.lookupRequired Env.repoFixturesDir + let + registryOrigin = Path.concat [ fixturesDir, "purescript", "registry" ] + registryIndexOrigin = Path.concat [ fixturesDir, "purescript", "registry-index" ] + createStashTag registryOrigin + createStashTag registryIndexOrigin + Console.log "Stashed git fixtures at post-startup tag" + where + createStashTag dir = do + -- Delete existing tag if present, then create new one at HEAD + void $ liftAff $ Git.gitCLI [ "tag", "-d", "post-startup" ] (Just dir) + void $ gitOrFail [ "tag", "post-startup" ] dir + -- | Clear server log files for test isolation. -- | Deletes *.log files from the scratch/logs directory but preserves the directory itself. resetLogs :: E2E Unit @@ -246,6 +281,26 @@ waitForAllMatrixJobs pkg = go 120 0 liftAff $ Aff.delay (Milliseconds 1000.0) go (attempts - 1) totalCount +-- | Wait for all pending jobs (of any type) to complete. +-- | Useful for ensuring startup jobs finish before running tests that clear the DB. +waitForAllPendingJobs :: E2E Unit +waitForAllPendingJobs = go 300 -- 5 minutes max + where + go :: Int -> E2E Unit + go 0 = liftAff $ Aff.throwError $ Aff.error "Timed out waiting for all jobs to complete" + go attempts = do + jobs <- Client.getJobs + let + pendingJobs = Array.filter (\j -> isNothing (V1.jobInfo j).finishedAt) jobs + pendingCount = Array.length pendingJobs + if pendingCount == 0 then + pure unit + else do + when (attempts `mod` 30 == 0) do + Console.log $ "Waiting for " <> show pendingCount <> " pending jobs to complete..." + liftAff $ Aff.delay (Milliseconds 1000.0) + go (attempts - 1) + -- | Check if a job is a matrix job for the given package. isMatrixJobFor :: PackageFixture -> Job -> Boolean isMatrixJobFor pkg = case _ of diff --git a/app-e2e/src/Test/E2E/Support/Fixtures.purs b/app-e2e/src/Test/E2E/Support/Fixtures.purs index 7fe0b556..b9b4bfe2 100644 --- a/app-e2e/src/Test/E2E/Support/Fixtures.purs +++ b/app-e2e/src/Test/E2E/Support/Fixtures.purs @@ -5,9 +5,11 @@ module Test.E2E.Support.Fixtures , effect , console , prelude + , unsafeCoerce , effectPublishData , effectPublishDataDifferentLocation , consolePublishData + , unsafeCoercePublishData , failingTransferData , nonexistentTransferData , trusteeAuthenticatedData @@ -99,6 +101,22 @@ consolePublishData = , version: console.version } +-- | Publish data for unsafe-coerce@6.0.0, used by package set tests. +-- | Has no dependencies. Published first to create the tarball before adding to package set. +unsafeCoercePublishData :: Operation.PublishData +unsafeCoercePublishData = + { name: unsafeCoerce.name + , location: Just $ GitHub + { owner: "purescript" + , repo: "purescript-unsafe-coerce" + , subdir: Nothing + } + , ref: "v6.0.0" + , compiler: Utils.unsafeVersion "0.15.10" + , resolutions: Nothing + , version: unsafeCoerce.version + } + -- | Unpublish data for effect@4.0.0, used for publish-then-unpublish tests. effectUnpublishData :: UnpublishData effectUnpublishData = @@ -222,11 +240,11 @@ signTransfer privateKey transferData = do , signature } --- | type-equality@4.0.1 fixture package (exists in registry-index but not in initial package set) -typeEquality :: PackageFixture -typeEquality = { name: Utils.unsafePackageName "type-equality", version: Utils.unsafeVersion "4.0.1" } +-- | unsafe-coerce@6.0.0 fixture package (exists in registry-index but not in package set) +unsafeCoerce :: PackageFixture +unsafeCoerce = { name: Utils.unsafePackageName "unsafe-coerce", version: Utils.unsafeVersion "6.0.0" } --- | Package set request to add type-equality@4.0.1. +-- | Package set request to add unsafe-coerce@6.0.0. -- | This is an unauthenticated request (no signature) since adding packages -- | doesn't require trustee authentication. packageSetAddRequest :: PackageSetUpdateRequest @@ -234,7 +252,7 @@ packageSetAddRequest = let payload = PackageSetUpdate { compiler: Nothing - , packages: Map.singleton typeEquality.name (Just typeEquality.version) + , packages: Map.singleton unsafeCoerce.name (Just unsafeCoerce.version) } rawPayload = JSON.print $ CJ.encode Operation.packageSetOperationCodec payload in diff --git a/app-e2e/src/Test/Main.purs b/app-e2e/src/Test/Main.purs index 310cc27f..3f69070a 100644 --- a/app-e2e/src/Test/Main.purs +++ b/app-e2e/src/Test/Main.purs @@ -10,7 +10,7 @@ import Test.E2E.Endpoint.Scheduler as Scheduler import Test.E2E.Endpoint.Transfer as Transfer import Test.E2E.Endpoint.Unpublish as Unpublish import Test.E2E.GitHubIssue as GitHubIssue -import Test.E2E.Support.Env (assertReposClean, mkTestEnv, resetTestState, runE2E) +import Test.E2E.Support.Env (assertReposClean, mkTestEnv, resetTestState, runE2E, stashGitFixtures, waitForAllPendingJobs) import Test.E2E.Workflow as Workflow import Test.Spec (hoistSpec) import Test.Spec as Spec @@ -26,6 +26,15 @@ main = do -- so we need to run these tests without cleaning out the state first Spec.describe "Scheduler" Scheduler.spec + -- After scheduler tests, wait for startup jobs to complete and stash the + -- git fixtures state. This ensures that subsequent tests can reset to + -- a state where startup jobs (like new compiler matrix jobs) have already + -- updated the metadata. + Spec.describe "Setup" do + Spec.it "waits for startup jobs and stashes fixtures" do + waitForAllPendingJobs + stashGitFixtures + Spec.before_ resetTestState $ Spec.after_ assertReposClean $ Spec.describe "E2E Tests" do Spec.describe "Endpoints" do Spec.describe "Publish" Publish.spec diff --git a/app/fixtures/github-packages/unsafe-coerce-6.0.0/bower.json b/app/fixtures/github-packages/unsafe-coerce-6.0.0/bower.json new file mode 100644 index 00000000..eb6293c5 --- /dev/null +++ b/app/fixtures/github-packages/unsafe-coerce-6.0.0/bower.json @@ -0,0 +1,21 @@ +{ + "name": "purescript-unsafe-coerce", + "homepage": "https://github.com/purescript/purescript-unsafe-coerce", + "license": "BSD-3-Clause", + "repository": { + "type": "git", + "url": "https://github.com/purescript/purescript-unsafe-coerce.git" + }, + "ignore": [ + "**/.*", + "bower_components", + "node_modules", + "output", + "test", + "bower.json", + "package.json" + ], + "devDependencies": { + "purescript-console": "^6.0.0" + } +} diff --git a/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.js b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.js new file mode 100644 index 00000000..6c7317ae --- /dev/null +++ b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.js @@ -0,0 +1,5 @@ +// module Unsafe.Coerce + +export const unsafeCoerce = function (x) { + return x; +}; diff --git a/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.purs b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.purs new file mode 100644 index 00000000..c38fd4be --- /dev/null +++ b/app/fixtures/github-packages/unsafe-coerce-6.0.0/src/Unsafe/Coerce.purs @@ -0,0 +1,26 @@ +module Unsafe.Coerce + ( unsafeCoerce + ) where + +-- | A _highly unsafe_ function, which can be used to persuade the type system that +-- | any type is the same as any other type. When using this function, it is your +-- | (that is, the caller's) responsibility to ensure that the underlying +-- | representation for both types is the same. +-- | +-- | Because this function is extraordinarily flexible, type inference +-- | can greatly suffer. It is highly recommended to define specializations of +-- | this function rather than using it as-is. For example: +-- | +-- | ```purescript +-- | fromBoolean :: Boolean -> Json +-- | fromBoolean = unsafeCoerce +-- | ``` +-- | +-- | This way, you won't have any nasty surprises due to the inferred type being +-- | different to what you expected. +-- | +-- | After the v0.14.0 PureScript release, some of what was accomplished via +-- | `unsafeCoerce` can now be accomplished via `coerce` from +-- | `purescript-safe-coerce`. See that library's documentation for more +-- | context. +foreign import unsafeCoerce :: forall a b. a -> b diff --git a/app/fixtures/registry-index/tr/an/transferred b/app/fixtures/registry-index/tr/an/transferred deleted file mode 100644 index 21889f95..00000000 --- a/app/fixtures/registry-index/tr/an/transferred +++ /dev/null @@ -1 +0,0 @@ -{"name":"transferred","version":"1.0.0","license":"MIT","location":{"githubOwner":"old-owner","githubRepo":"purescript-transferred"},"ref":"v1.0.0","dependencies":{}} diff --git a/app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz b/app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..34451628ffbbc56bec9e79b41277f5b85877497d GIT binary patch literal 1310 zcmV+(1>yQ1iwFP!000001MOOEZ`(Ey&S(D$B43gQu`Kx&NRa_ei(%`R70ZTw7`B*} zP8K_f6iCW1+w$Lcq-;laoqBd$ZH4&&jvexJcf8~Ap-FqK4|Mh&&*0F7vq8{{m&)5Iv=O75$oioxqwjDWppTIcGf)fzLM6eBMbS?lTH*5~!4ECjBsn!EdV%}(=e zN(;5w2*V`H_l<3p66B?jaeaVDr9=N8=g^wdLFGsHs(Sa{lAu~udZ6&)zXeF zg$LA|3l3bXWjhxUJY*s&QdqT#Lg&TGLk&u=X|V;hY{_Z(2Zpk2!=~S?D-lfIK@L0u z9x_zCia=!~q2NlSHXgs*a6`#-n-e|fm||FrR3{g*|592(ev{qF=F zcmG!lcI3Z818>W-(~oLz8~d;S1K-^Ly7m9d!>#|PjUDxWc+pp|tNzW8J=W{L+wb?> z`ClDqwMco5MaqnOWyTu~nB_uhVy>GsDQ8L9MQ%x4@X(y_k;~aDd&Vb07m}0N>&D}2 zw}yO8*qi^&<-ndc4&eW;@4o-l0>}TSjep1g2NZn;JL|7MG)Ocx> zW7Dx`8xI8t6ZM;oOk2{}FlT-%FH5(`O2 z=#%wGRUP)MeEHBFgyvFo-Af}3GcBo*5#v-cjA5L@jEz#bUW}i|7-^1Sw`xoz#}cSU zNmazy=aKL2NEa!P5Q;1_m!CG&5yXt6H#vlirtDAavLYfb)vbFYX&KpsE@O&8q^il3 zN4~{T!i;8l3OAKu4Tr<03)-?NmLj_oA_dfPb3>LNo8OT4O7Kr*Zr5kmQf-ppZ4)@*;7h{bBP?#wtaCVBgR5dlzG&H{wOPNa0`h?NN zsHy`U+FWumHI=a`Pln+_$V6)Kbmbau18ynzE6ka(;nP5Mhz6P0>uqYmU0828QqyPW_ literal 0 HcmV?d00001 diff --git a/app/fixtures/registry/metadata/prelude.json b/app/fixtures/registry/metadata/prelude.json index 8c14057a..0b404725 100644 --- a/app/fixtures/registry/metadata/prelude.json +++ b/app/fixtures/registry/metadata/prelude.json @@ -7,8 +7,7 @@ "6.0.1": { "bytes": 31129, "compilers": [ - "0.15.10", - "0.15.11" + "0.15.10" ], "hash": "sha256-EbbFV0J5xV0WammfgCv6HRFSK7Zd803kkofE8aEoam0=", "publishedTime": "2022-08-18T20:04:00.000Z", diff --git a/app/fixtures/registry/metadata/transferred.json b/app/fixtures/registry/metadata/transferred.json deleted file mode 100644 index b1972993..00000000 --- a/app/fixtures/registry/metadata/transferred.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "location": { - "githubOwner": "old-owner", - "githubRepo": "purescript-transferred" - }, - "published": { - "1.0.0": { - "bytes": 1000, - "compilers": ["0.15.10"], - "hash": "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "publishedTime": "2022-01-01T00:00:00.000Z", - "ref": "v1.0.0" - } - }, - "unpublished": {} -} diff --git a/app/fixtures/registry/metadata/type-equality.json b/app/fixtures/registry/metadata/type-equality.json index e51b5261..35c13b75 100644 --- a/app/fixtures/registry/metadata/type-equality.json +++ b/app/fixtures/registry/metadata/type-equality.json @@ -1,6 +1,6 @@ { "location": { - "githubOwner": "purescript", + "githubOwner": "old-owner", "githubRepo": "purescript-type-equality" }, "published": { diff --git a/nix/test/config.nix b/nix/test/config.nix index de5ff3e1..233eb846 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -172,6 +172,55 @@ let }; }; + # Unsafe-coerce package helpers (unsafe-coerce@6.0.0) + unsafeCoerceBase64Response = + fileName: + base64Response { + url = "/repos/purescript/purescript-unsafe-coerce/contents/${fileName}?ref=v6.0.0"; + inherit fileName; + filePath = rootPath + "/app/fixtures/github-packages/unsafe-coerce-6.0.0/${fileName}"; + }; + + unsafeCoerce404Response = fileName: { + request = { + method = "GET"; + url = "/repos/purescript/purescript-unsafe-coerce/contents/${fileName}?ref=v6.0.0"; + }; + response = { + status = 404; + headers."Content-Type" = "application/json"; + jsonBody = { + message = "Not Found"; + documentation_url = "https://docs.github.com/rest/repos/contents#get-repository-content"; + }; + }; + }; + + # Type-equality package helpers (type-equality@4.0.2) + # Note: Uses purescript owner (actual location) not old-owner (metadata location) + typeEqualityBase64Response = + fileName: + base64Response { + url = "/repos/purescript/purescript-type-equality/contents/${fileName}?ref=v4.0.2"; + inherit fileName; + filePath = rootPath + "/app/fixtures/github-packages/type-equality-4.0.1/${fileName}"; + }; + + typeEquality404Response = fileName: { + request = { + method = "GET"; + url = "/repos/purescript/purescript-type-equality/contents/${fileName}?ref=v4.0.2"; + }; + response = { + status = 404; + headers."Content-Type" = "application/json"; + jsonBody = { + message = "Not Found"; + documentation_url = "https://docs.github.com/rest/repos/contents#get-repository-content"; + }; + }; + }; + # GitHub API wiremock mappings githubMappings = [ (effectBase64Response "bower.json") @@ -188,6 +237,20 @@ let (console404Response "spago.dhall") (console404Response "purs.json") (console404Response "package.json") + # Unsafe-coerce package (unsafe-coerce@6.0.0) + (unsafeCoerceBase64Response "bower.json") + (unsafeCoerce404Response "LICENSE") + (unsafeCoerce404Response "spago.yaml") + (unsafeCoerce404Response "spago.dhall") + (unsafeCoerce404Response "purs.json") + (unsafeCoerce404Response "package.json") + # Type-equality package (type-equality@4.0.2 for legacy imports test) + (typeEqualityBase64Response "bower.json") + (typeEqualityBase64Response "LICENSE") + (typeEquality404Response "spago.yaml") + (typeEquality404Response "spago.dhall") + (typeEquality404Response "purs.json") + (typeEquality404Response "package.json") { request = { method = "GET"; @@ -205,8 +268,7 @@ let }; }; } - # Tags for prelude package (used by Scheduler tests) - # Includes v6.0.1 (already published) and v6.0.2 (new version for scheduler to discover) + # Tags for prelude package (only v6.0.1 which is already published) { request = { method = "GET"; @@ -223,21 +285,16 @@ let url = "https://api.github.com/repos/purescript/purescript-prelude/commits/abc123def456"; }; } - { - name = "v6.0.2"; - commit = { - sha = "def456abc789"; - url = "https://api.github.com/repos/purescript/purescript-prelude/commits/def456abc789"; - }; - } ]; }; } - # Tags for type-equality package + # Tags for type-equality package (used by two scheduler tests): + # 1. Transfer detection: metadata says old-owner, commit URLs point to purescript + # 2. Legacy imports: v4.0.2 is a new version not yet published { request = { method = "GET"; - url = "/repos/purescript/purescript-type-equality/tags"; + url = "/repos/old-owner/purescript-type-equality/tags"; }; response = { status = 200; @@ -247,29 +304,16 @@ let name = "v4.0.1"; commit = { sha = "type-eq-sha-401"; + # Points to actual owner - scheduler detects this transfer url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-401"; }; } - ]; - }; - } - # Tags for transferred package (scheduler transfer detection test) - # Metadata says old-owner but tags point to new-owner - triggers transfer detection - { - request = { - method = "GET"; - url = "/repos/old-owner/purescript-transferred/tags"; - }; - response = { - status = 200; - headers."Content-Type" = "application/json"; - jsonBody = [ { - name = "v1.0.0"; + name = "v4.0.2"; commit = { - sha = "transferred-sha-100"; - # Points to NEW owner - scheduler should detect this transfer - url = "https://api.github.com/repos/new-owner/purescript-transferred/commits/transferred-sha-100"; + sha = "type-eq-sha-402"; + # New version not yet published - scheduler detects for legacy import + url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-402"; }; } ]; @@ -873,6 +917,8 @@ let cp -r ${rootPath}/app/fixtures/{registry-index,registry,package-sets} "$FIXTURES_DIR/purescript/" cp -r ${rootPath}/app/fixtures/github-packages/effect-4.0.0 "$FIXTURES_DIR/purescript/purescript-effect" cp -r ${rootPath}/app/fixtures/github-packages/console-6.1.0 "$FIXTURES_DIR/purescript/purescript-console" + cp -r ${rootPath}/app/fixtures/github-packages/unsafe-coerce-6.0.0 "$FIXTURES_DIR/purescript/purescript-unsafe-coerce" + cp -r ${rootPath}/app/fixtures/github-packages/type-equality-4.0.1 "$FIXTURES_DIR/purescript/purescript-type-equality" chmod -R u+w "$FIXTURES_DIR/purescript" # Set type-equality publishedTime to current time for package set update test @@ -896,6 +942,9 @@ let gitbot -C "$FIXTURES_DIR/purescript/package-sets" tag -m "psc-0.15.9-20230105" psc-0.15.9-20230105 gitbot -C "$FIXTURES_DIR/purescript/purescript-effect" tag -m "v4.0.0" v4.0.0 gitbot -C "$FIXTURES_DIR/purescript/purescript-console" tag -m "v6.1.0" v6.1.0 + gitbot -C "$FIXTURES_DIR/purescript/purescript-unsafe-coerce" tag -m "v6.0.0" v6.0.0 + gitbot -C "$FIXTURES_DIR/purescript/purescript-type-equality" tag -m "v4.0.1" v4.0.1 + gitbot -C "$FIXTURES_DIR/purescript/purescript-type-equality" tag -m "v4.0.2" v4.0.2 ''; }; From 0ee925ce00bc6edf71ad4887f7d53c34e76b97ff Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Fri, 16 Jan 2026 02:51:45 +0200 Subject: [PATCH 04/20] Apply review feedback --- app-e2e/src/Test/E2E/Endpoint/Startup.purs | 10 +- app-e2e/src/Test/Main.purs | 4 +- .../type-equality-4.0.2.tar.gz | Bin 0 -> 2184 bytes app/src/App/Effect/Registry.purs | 105 +++++++++++++++++- app/src/App/Server/Env.purs | 5 + app/src/App/Server/Scheduler.purs | 93 +++++++++++----- nix/test/config.nix | 26 +++-- scripts/src/ArchiveSeeder.purs | 13 ++- scripts/src/CompilerVersions.purs | 3 + scripts/src/LegacyImporter.purs | 13 ++- scripts/src/PackageDeleter.purs | 3 + scripts/src/Solver.purs | 13 ++- scripts/src/VerifyIntegrity.purs | 3 + 13 files changed, 245 insertions(+), 46 deletions(-) create mode 100644 app/fixtures/registry-storage/type-equality-4.0.2.tar.gz diff --git a/app-e2e/src/Test/E2E/Endpoint/Startup.purs b/app-e2e/src/Test/E2E/Endpoint/Startup.purs index db804508..eb17e02c 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Startup.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Startup.purs @@ -1,5 +1,5 @@ -- | E2E tests for the Scheduler and JobExecutor startup, covering: --- | - scheduleLegacyImports: Detects new package versions via GitHub tags +-- | - scheduleDailyPublish: Detects new package versions via GitHub tags -- | - scheduleTransfers: Detects packages that moved to new GitHub locations -- | - schedulePackageSetUpdates: Detects recent uploads for package set inclusion -- | - checkIfNewCompiler: Detects new compiler and enqueues matrix jobs @@ -25,7 +25,7 @@ import Test.Spec as Spec spec :: E2ESpec spec = do - Spec.describe "scheduleLegacyImports" do + Spec.describe "scheduleDailyPublish" do Spec.it "enqueues publish jobs for new package versions discovered via GitHub tags" do -- The scheduler runs at server startup and should have already -- fetched tags for packages in the registry metadata. @@ -46,8 +46,10 @@ spec = do case typeEqualityJob of Just (PublishJob { payload }) -> do - -- Verify the compiler is from previous version (type-equality@4.0.1 has compilers [0.15.10, 0.15.11]) - -- The scheduler should use the lowest compiler from the previous version for compatibility + -- The scheduler determines a compatible compiler by looking at the previous + -- version's dependencies and finding the intersection of their supported compilers. + -- type-equality@4.0.1 has no dependencies, so the scheduler falls back to the + -- lowest compiler from the previous version (0.15.10). let expectedCompiler = unsafeFromRight (Version.parse "0.15.10") when (payload.compiler /= expectedCompiler) do Assert.fail $ "Expected compiler 0.15.10 but got " <> Version.print payload.compiler diff --git a/app-e2e/src/Test/Main.purs b/app-e2e/src/Test/Main.purs index 3f69070a..5a81fbfd 100644 --- a/app-e2e/src/Test/Main.purs +++ b/app-e2e/src/Test/Main.purs @@ -6,7 +6,7 @@ import Data.Time.Duration (Milliseconds(..)) import Test.E2E.Endpoint.Jobs as Jobs import Test.E2E.Endpoint.PackageSets as PackageSets import Test.E2E.Endpoint.Publish as Publish -import Test.E2E.Endpoint.Scheduler as Scheduler +import Test.E2E.Endpoint.Startup as Startup import Test.E2E.Endpoint.Transfer as Transfer import Test.E2E.Endpoint.Unpublish as Unpublish import Test.E2E.GitHubIssue as GitHubIssue @@ -24,7 +24,7 @@ main = do runSpecAndExitProcess' config [ consoleReporter ] $ hoistE2E env do -- The scheduler runs at startup and enqueues a bunch of jobs in the DB, -- so we need to run these tests without cleaning out the state first - Spec.describe "Scheduler" Scheduler.spec + Spec.describe "Startup" Startup.spec -- After scheduler tests, wait for startup jobs to complete and stash the -- git fixtures state. This ensures that subsequent tests can reset to diff --git a/app/fixtures/registry-storage/type-equality-4.0.2.tar.gz b/app/fixtures/registry-storage/type-equality-4.0.2.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..ba7126b6006c7f849f7252685f75bb22467248a9 GIT binary patch literal 2184 zcmV;32zU1%iwFP!000001MOLRbK*u4-@o-K`n+{1vj@U}A1UXYlnk=1Dh!+?cH`Ph zVj%|5S!fiEWXg4y?|$7gU>=9pSugQjF7vBhGnz+tPd}#Tkw2uY!hSA-ILaR?hvtD< ztG?MW4iG0tNAef3`MXj+Y}D#luGJ4uYH#3Z*Me+AA@U$Y_Fqb8b8^2E}$Y$44g{_HP{34&K14DDe+&|0Yi#vS>WXp?*;N8hS;>yfBN>+%Q}= zj6|MAgCdW30zoo_LNJID5WL7jCTj*!5@ZiB;@MP`VHV{R;Fq4HAP36F=^A>qqTQteSLNNPooXTq2U4=`ZlAt*lwmJG3$ku#tm zQ=T(WTETHhFuKQXBW#eiKqHRw8CiPnIEF}BNS=d(BJ!ta?)7T*Bz73b%Zc8fK zv)%WYalCJgcGr{I>U(w>Irx@jj3iK1-ZxY}?|Y6^n$z)Z*X{Rwr`svv%s-%7P(}-f zwxo8u9XVChP}jX7qND?<{4#vFva#MJg-T6Xqyi5$*7R4qAwN{0zcNSY*q3eR((W{E zYV8u04~}P-&{mE|9grlk{a_)_zML}I2&Ga*$1@flTb7mt=K`$OdxylHcShTL&fHI= z%9>Ymb(_Y{*Z*z#-?gpQwQWv^JL2(7{;xGojyLoF32q5={(l9$g%nf1kSV{i<_w1M z@lPMym`-F$i&-Gp5BrlmPfx4WXgW5c)>`NNW}9Ziidz!%nR{iiZhg5j8vgx?LaAPFaD(?DQW zeYw0rD0BOFKNl=}I9u@gbHi#ne^D&3JZC~v!d;jsqI}6ORTbuvgUPdTH4VbMV9dnR zoaIzXUrwlm=@!3X@lClOHxszg%7tOPeG3kL25~HRB!+Q&dy6SN7<1gQ$a+|JEo!Gx zt<{Z;VO-+|A>Q2H;P&8SS`2WjG{HRsR;+2~OXl<&ET_LaY+L^ae1`koe+ZuJhQ~AO z|G~-8=K6nlRIBUt|26PW17MGK^?RqV_f(|pm8sW+PZ`Fv?D?{VP|=C;{F4Yt=G<#l z8kJ@o;ITn9;^B#l2u~Cb*iOP>9nqlP$V_&^sv?VDE`~{jkj^iXV>HIY4blIoNWXhm zHQz1x$WZtR>83ov6-g}W6FhGIG?mNMV;!Db(&AA<%A0FcWI!!y=8Kc6!>S*oC_{GN zVksV-SQ17^_)!BZGogi-$5J0Z9%Hy3Jd#}YN!V_{X5%jQ_?_O~?OhU@iY| zN!Ir)qw6+mmCo@iQh3bOh3_&xReJ8u5|*uRGZulUI=GnoKe-KASqavcfj?N#?hxDJ zUu5A<-vHGf|8w5|8g>$5ZyF)BdiU;PdT&SU=G3zpsK%+TUIrm}p1vavsm* z|Jre*v0?w?liH!q|E~dch6OUWxiB|o*nJ4&KnPhcs|8k;=?RaIWGUzJho11TP=eWn zWy~-lJSb;5^l)cwHaVl0*?mPxeno?rl?(&V0RIA?9{$Mv=B)5P%W^S)4#s)`x52G^ z54|1SsxkCLD4o@m-wQ$vMnxj;Sc4c=_XdxzsUk|oF;o5Jm1KgF4Oq-)FpJ`tUT^2` zwfT%Ise)`QZy|*mm2)VPX!JlcOr%He;jsR*6*x!5;?;^)8E?l&Zvp7x7RHN_bB1u0uA85;mk4Vmwu4 z$bt#e*fb)Xp3+q5LiWytJE(lC{xlzNbBzsaK>!2z>)9#|szFoGxIr1q_#4da56VMU z*rX<;sxKY@(LWaIps7-~HM*N*0yu-fFy_}8;B(3Qj0*dTKROc{X2}lQ%}rU)zMTt8 zUC$s41e?>}CqjRhjX|kIO2eU6q~FX)w8@s-7WHDlx~->;wLt-7nu>)aK&HU6?{ zF2R3$Y0%Kn(9qD((9qD((9qD((9qD((9qD((9qD((9qD((9qD((9qD((9rlF#J>Rp KEZV;SPyhfn&`zfS literal 0 HcmV?d00001 diff --git a/app/src/App/Effect/Registry.purs b/app/src/App/Effect/Registry.purs index 48fbdf4a..4d34f51e 100644 --- a/app/src/App/Effect/Registry.purs +++ b/app/src/App/Effect/Registry.purs @@ -18,10 +18,13 @@ import Data.Set as Set import Data.String as String import Data.Time.Duration as Duration import Effect.Aff as Aff +import Effect.Aff.AVar (AVar) +import Effect.Aff.AVar as AVar import Effect.Ref as Ref import JSON as JSON import Node.FS.Aff as FS.Aff import Node.Path as Path +import Registry.Foreign.FSExtra as FS.Extra import Registry.App.CLI.Git (GitResult) import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache (class MemoryEncodable, Cache, CacheRef, MemoryEncoding(..)) @@ -155,6 +158,103 @@ data RepoKey | ManifestIndexRepo | LegacyPackageSetsRepo +derive instance Eq RepoKey +derive instance Ord RepoKey + +-- | Identifies which process is using the registry, for lock ownership tracking. +data Process + = Scheduler + | JobExecutor + | API + | ScriptLegacyImporter + | ScriptPackageDeleter + | ScriptSolver + | ScriptVerifyIntegrity + | ScriptCompilerVersions + | ScriptArchiveSeeder + +derive instance Eq Process + +instance Show Process where + show Scheduler = "Scheduler" + show JobExecutor = "JobExecutor" + show API = "API" + show ScriptLegacyImporter = "ScriptLegacyImporter" + show ScriptPackageDeleter = "ScriptPackageDeleter" + show ScriptSolver = "ScriptSolver" + show ScriptVerifyIntegrity = "ScriptVerifyIntegrity" + show ScriptCompilerVersions = "ScriptCompilerVersions" + show ScriptArchiveSeeder = "ScriptArchiveSeeder" + +-- | A lock for a single repository, tracking both the mutex and the owner. +type RepoLock = { lock :: AVar Unit, owner :: Ref (Maybe Process) } + +-- | Per-repository locks to prevent concurrent access. +type RepoLocks = Ref (Map RepoKey RepoLock) + +-- | Create a new empty set of repo locks. +newRepoLocks :: forall m. MonadEffect m => m RepoLocks +newRepoLocks = liftEffect $ Ref.new Map.empty + +-- | Get or create a lock for a repository. +getOrCreateLock :: RepoLocks -> RepoKey -> Aff RepoLock +getOrCreateLock locksRef key = do + locks <- liftEffect $ Ref.read locksRef + case Map.lookup key locks of + Just lock -> pure lock + Nothing -> do + lock <- AVar.new unit + owner <- liftEffect $ Ref.new Nothing + let repoLock = { lock, owner } + liftEffect $ Ref.modify_ (Map.insert key repoLock) locksRef + pure repoLock + +-- | Acquire a repository lock, run an action, and release the lock. +-- | The lock prevents concurrent access to the same repository. +withRepoLock + :: forall r a + . Process + -> RepoLocks + -> RepoKey + -> Run (LOG + AFF + EFFECT + r) a + -> Run (LOG + AFF + EFFECT + r) a +withRepoLock process locks key action = do + repoLock <- Run.liftAff $ getOrCreateLock locks key + Run.liftAff $ AVar.take repoLock.lock + Run.liftEffect $ Ref.write (Just process) repoLock.owner + result <- action + Run.liftEffect $ Ref.write Nothing repoLock.owner + Run.liftAff $ AVar.put unit repoLock.lock + pure result + +-- | Clear any locks owned by a specific process. +-- | Used to clean up orphaned locks when a process crashes and restarts. +clearOwnLocks :: forall r. Process -> RepoLocks -> Run (LOG + AFF + EFFECT + r) Unit +clearOwnLocks process locksRef = do + locks <- Run.liftEffect $ Ref.read locksRef + for_ (Map.toUnfoldable locks :: Array _) \(Tuple _ repoLock) -> do + owner <- Run.liftEffect $ Ref.read repoLock.owner + when (owner == Just process) do + Log.warn $ "Clearing orphaned lock for " <> show process + Run.liftEffect $ Ref.write Nothing repoLock.owner + -- Put the unit back to release the lock + Run.liftAff $ AVar.put unit repoLock.lock + +-- | Validate that a repository is in a valid state. +-- | If the repo is corrupted (e.g., from an interrupted clone), delete it. +validateRepo :: forall r. FilePath -> Run (LOG + AFF + EFFECT + r) Unit +validateRepo path = do + exists <- Run.liftAff $ Aff.attempt (FS.Aff.stat path) + case exists of + Left _ -> pure unit -- Doesn't exist, nothing to validate + Right _ -> do + result <- Run.liftAff $ Git.gitCLI [ "rev-parse", "--git-dir" ] (Just path) + case result of + Left _ -> do + Log.warn $ "Detected corrupted repo at " <> path <> ", deleting" + Run.liftAff $ FS.Extra.remove path + Right _ -> pure unit + -- | A legend for values that can be committed. We know where each kind of value -- | ought to exist, so we can create a correct path for any given type ourselves. data CommitKey @@ -207,6 +307,8 @@ type RegistryEnv = , write :: WriteMode , debouncer :: Debouncer , cacheRef :: CacheRef + , repoLocks :: RepoLocks + , process :: Process } type Debouncer = Ref (Map FilePath DateTime) @@ -705,8 +807,9 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << -- | Get the repository at the given key, recording whether the pull or clone -- | had any effect (ie. if the repo was already up-to-date). + -- | Uses per-repository locking to prevent race conditions during clone. pull :: RepoKey -> Run _ (Either String GitResult) - pull repoKey = do + pull repoKey = withRepoLock env.process env.repoLocks repoKey do let path = repoPath repoKey address = repoAddress repoKey diff --git a/app/src/App/Server/Env.purs b/app/src/App/Server/Env.purs index 70e5698f..89fb0b8a 100644 --- a/app/src/App/Server/Env.purs +++ b/app/src/App/Server/Env.purs @@ -76,6 +76,7 @@ type ServerEnv = , octokit :: Octokit , vars :: ServerEnvVars , debouncer :: Registry.Debouncer + , repoLocks :: Registry.RepoLocks , db :: SQLite , jobId :: Maybe JobId } @@ -94,6 +95,7 @@ createServerEnv = do octokit <- Octokit.newOctokit vars.token vars.resourceEnv.githubApiUrl debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks db <- liftEffect $ SQLite.connect { database: vars.resourceEnv.databaseUrl.path @@ -111,6 +113,7 @@ createServerEnv = do pure { debouncer + , repoLocks , githubCacheRef , legacyCacheRef , registryCacheRef @@ -159,6 +162,8 @@ runEffects env operation = Aff.attempt do , workdir: scratchDir , debouncer: env.debouncer , cacheRef: env.registryCacheRef + , repoLocks: env.repoLocks + , process: Registry.API } ) # Archive.interpret Archive.handle diff --git a/app/src/App/Server/Scheduler.purs b/app/src/App/Server/Scheduler.purs index 273abb5b..1d1b2d5d 100644 --- a/app/src/App/Server/Scheduler.purs +++ b/app/src/App/Server/Scheduler.purs @@ -9,6 +9,8 @@ import Data.Array.NonEmpty as NonEmptyArray import Data.DateTime as DateTime import Data.Map as Map import Data.Set as Set +import Data.Set.NonEmpty (NonEmptySet) +import Data.Set.NonEmpty as NonEmptySet import Data.String as String import Data.Time.Duration (Hours(..)) import Effect.Aff (Milliseconds(..)) @@ -28,6 +30,7 @@ import Registry.Location (Location(..)) import Registry.Operation as Operation import Registry.PackageName as PackageName import Registry.PackageSet (PackageSet(..)) +import Registry.Range as Range import Run (Run) -- | The scheduler loop runs immediately, then every 24 hours. @@ -38,13 +41,13 @@ runScheduler env = runEffects env do Log.info "Starting Scheduler" loop where - sleepTime = Milliseconds (1000.0 * 60.0 * 60.0 * 12.0) + sleepTime = Milliseconds (1000.0 * 60.0 * 60.0 * 24.0) loop = do -- Run all scheduling checks scheduleTransfers schedulePackageSetUpdates - scheduleLegacyImports + scheduleDailyPublish Log.info "Scheduler cycle complete, sleeping for 24 hours..." -- Sleep for a while, then run again liftAff $ Aff.delay sleepTime @@ -185,24 +188,20 @@ findRecentUploads limit = do let getLatestRecentVersion :: Metadata -> Maybe Version - getLatestRecentVersion (Metadata metadata) = + getLatestRecentVersion (Metadata metadata) = do let - recentVersions = Array.catMaybes $ flip map (Map.toUnfoldable metadata.published) \(Tuple version { publishedTime }) -> - let - diff = DateTime.diff now publishedTime - in - if diff <= limit then Just version else Nothing - in - Array.last $ Array.sort recentVersions + recentVersions = Array.catMaybes $ flip map (Map.toUnfoldable metadata.published) + \(Tuple version { publishedTime }) -> if (DateTime.diff now publishedTime) <= limit then Just version else Nothing + Array.last $ Array.sort recentVersions pure $ Map.fromFoldable $ Array.catMaybes $ flip map (Map.toUnfoldable allMetadata) \(Tuple name metadata) -> map (Tuple name) $ getLatestRecentVersion metadata -- | Check for new tags on existing packages and enqueue publish jobs for -- | versions not yet published. This allows the registry to automatically --- | import new versions of packages that only have legacy manifests. -scheduleLegacyImports :: Run ServerEffects Unit -scheduleLegacyImports = do +-- | publish new versions of packages that are already in the registry. +scheduleDailyPublish :: Run ServerEffects Unit +scheduleDailyPublish = do Log.info "Scheduler: checking for new package versions..." allMetadata <- Registry.readAllMetadata @@ -234,35 +233,73 @@ scheduleLegacyImports = do else Just { version, ref: tag.name } for_ newVersions \{ version, ref } -> - enqueuePublishJob name (Metadata metadata) version ref - - -- Delay between packages to spread GitHub API load - liftAff $ Aff.delay (Milliseconds 500.0) + enqueuePublishJob allMetadata name (Metadata metadata) version ref -- | Enqueue a publish job for a new package version discovered by the scheduler. --- | Uses the lowest compiler from the previous published version for compatibility, --- | falling back to the latest compiler if no previous version exists. -enqueuePublishJob :: PackageName -> Metadata -> Version -> String -> Run ServerEffects Unit -enqueuePublishJob name (Metadata metadata) version ref = do +-- | Attempts to find a compatible compiler by looking at the previous version's +-- | dependencies. Falls back to the lowest compiler from the previous version if +-- | no dependencies exist, or to the latest compiler if no previous version exists. +enqueuePublishJob :: Map PackageName Metadata -> PackageName -> Metadata -> Version -> String -> Run ServerEffects Unit +enqueuePublishJob allMetadata name (Metadata metadata) version ref = do -- Check if a publish job already exists for this package version existingJob <- Db.selectPublishJob name version case existingJob of Just _ -> Log.debug $ "Publish job already exists for " <> formatPackageVersion name version Nothing -> do - -- Use the lowest compiler from previous version for compatibility, - -- falling back to latest if no previous version exists + -- Try to find a compatible compiler by looking at the previous version's dependencies compiler <- case Map.findMax metadata.published of - Just { value: publishedInfo } -> - pure $ NonEmptyArray.head publishedInfo.compilers - Nothing -> NonEmptyArray.last <$> PursVersions.pursVersions + Just { key: prevVersion, value: publishedInfo } -> do + -- Look up the manifest for the previous version to get its dependencies + maybeManifest <- Registry.readManifest name prevVersion + case maybeManifest of + Just (Manifest manifest) | not (Map.isEmpty manifest.dependencies) -> do + -- Use previous version's dependencies to find compatible compilers + -- Find the highest published version of each dependency within its range + let + depVersions :: Map PackageName Version + depVersions = Map.mapMaybeWithKey (\depName range -> + case Map.lookup depName allMetadata of + Just (Metadata depMeta) -> + Array.last $ Array.filter (Range.includes range) $ Array.sort $ Array.fromFoldable $ Map.keys depMeta.published + Nothing -> Nothing + ) manifest.dependencies + + case compatibleCompilers allMetadata depVersions of + Just compilerSet -> pure $ NonEmptySet.min compilerSet + -- No intersection found, fall back to lowest compiler from previous version + Nothing -> pure $ NonEmptyArray.head publishedInfo.compilers + -- No manifest or no dependencies, fall back to lowest compiler from previous version + _ -> pure $ NonEmptyArray.head publishedInfo.compilers + Nothing -> + NonEmptyArray.last <$> PursVersions.pursVersions let payload = { name - , location: Just metadata.location + -- Don't specify location - use current metadata location at publish time. + -- This avoids race conditions with transfer jobs that may update the location. + , location: Nothing , ref , version , compiler , resolutions: Nothing } jobId <- Db.insertPublishJob { payload } - Log.info $ "Enqueued legacy publish job " <> unwrap jobId <> " for " <> formatPackageVersion name version + Log.info $ "Enqueued publish job " <> unwrap jobId <> " for " <> formatPackageVersion name version + +-- | Given a set of package versions, determine the set of compilers that can be +-- | used for all packages by intersecting their supported compiler ranges. +compatibleCompilers :: Map PackageName Metadata -> Map PackageName Version -> Maybe (NonEmptySet Version) +compatibleCompilers allMetadata resolutions = do + let + associated :: Array { compilers :: NonEmptyArray Version } + associated = Map.toUnfoldableUnordered resolutions # Array.mapMaybe \(Tuple depName depVersion) -> do + Metadata depMeta <- Map.lookup depName allMetadata + published <- Map.lookup depVersion depMeta.published + Just { compilers: published.compilers } + + case Array.uncons associated of + Nothing -> Nothing + Just { head, tail: [] } -> Just $ NonEmptySet.fromFoldable1 head.compilers + Just { head, tail } -> do + let foldFn prev = Set.intersection prev <<< Set.fromFoldable <<< _.compilers + NonEmptySet.fromFoldable $ Array.foldl foldFn (Set.fromFoldable head.compilers) tail diff --git a/nix/test/config.nix b/nix/test/config.nix index 233eb846..afd6f187 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -395,10 +395,20 @@ let ) ); - # Metadata fixtures directory (to determine which packages are "published") + # Metadata fixtures directory (to determine which package versions are "published") metadataFixturesDir = rootPath + "/app/fixtures/registry/metadata"; metadataFiles = builtins.attrNames (builtins.readDir metadataFixturesDir); - publishedPackageNames = map (f: lib.removeSuffix ".json" f) metadataFiles; + + # Parse metadata files to get the actual published versions (not just package names) + # Returns a set like { "prelude-6.0.1" = true; "type-equality-4.0.1" = true; } + publishedVersions = lib.foldl' (acc: fileName: + let + packageName = lib.removeSuffix ".json" fileName; + metadata = builtins.fromJSON (builtins.readFile (metadataFixturesDir + "/${fileName}")); + versions = builtins.attrNames (metadata.published or {}); + in + acc // lib.genAttrs (map (v: "${packageName}-${v}") versions) (_: true) + ) {} metadataFiles; # ============================================================================ # UNIFIED STORAGE MAPPINGS WITH WIREMOCK SCENARIOS @@ -412,9 +422,9 @@ let # Scenario design: # - One scenario per package-version (e.g., "effect-4.0.0") # - WireMock scenarios always start at state "Started" - # - Published packages (has metadata): "Started" means Present (tarball available) + # - Published versions (version exists in metadata.published): "Started" means Present # - After DELETE, transitions to "Deleted" state (404 on GET) - # - Unpublished packages (no metadata): "Started" means Absent (tarball 404) + # - Unpublished versions (new version not in metadata): "Started" means Absent (404) # - After PUT upload, transitions to "Present" state # - After DELETE, transitions to "Deleted" state (404 on GET) # @@ -430,7 +440,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; tarPath = "/${pkg.name}/${pkg.version}.tar.gz"; in if isPublished then @@ -521,7 +531,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; escapedName = lib.replaceStrings [ "-" ] [ "\\-" ] pkg.name; listUrlPattern = "/\\?.*prefix=${escapedName}.*"; presentContents = ''${pkg.name}/${pkg.version}.tar.gz1000"abc123"''; @@ -606,7 +616,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; escapedVersion = lib.replaceStrings [ "." ] [ "\\." ] pkg.version; urlPattern = "/${pkg.name}/${escapedVersion}\\.tar\\.gz.*"; in @@ -732,7 +742,7 @@ let pkg: let scenario = "${pkg.name}-${pkg.version}"; - isPublished = builtins.elem pkg.name publishedPackageNames; + isPublished = publishedVersions ? "${pkg.name}-${pkg.version}"; versionsUrl = "/packages/purescript-${pkg.name}/available-versions"; publishedVersionsBody = ''[["${pkg.version}","https://pursuit.purescript.org/packages/purescript-${pkg.name}/${pkg.version}"]]''; in diff --git a/scripts/src/ArchiveSeeder.purs b/scripts/src/ArchiveSeeder.purs index fe0ae805..ca4fdcdd 100644 --- a/scripts/src/ArchiveSeeder.purs +++ b/scripts/src/ArchiveSeeder.purs @@ -95,7 +95,18 @@ main = launchAff_ do runAppEffects <- do debouncer <- Registry.newDebouncer - let registryEnv = { pull: Git.Autostash, write: Registry.ReadOnly, repos: Registry.defaultRepos, workdir: scratchDir, debouncer, cacheRef: registryCacheRef } + repoLocks <- Registry.newRepoLocks + let + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptArchiveSeeder + } token <- Env.lookupRequired Env.githubToken s3 <- lift2 { key: _, secret: _ } (Env.lookupRequired Env.spacesKey) (Env.lookupRequired Env.spacesSecret) diff --git a/scripts/src/CompilerVersions.purs b/scripts/src/CompilerVersions.purs index 127d0b97..176a200a 100644 --- a/scripts/src/CompilerVersions.purs +++ b/scripts/src/CompilerVersions.purs @@ -119,6 +119,7 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -128,6 +129,8 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptCompilerVersions } -- Logging diff --git a/scripts/src/LegacyImporter.purs b/scripts/src/LegacyImporter.purs index 05e73ae8..2decb34b 100644 --- a/scripts/src/LegacyImporter.purs +++ b/scripts/src/LegacyImporter.purs @@ -189,7 +189,18 @@ main = launchAff_ do -- uploaded and manifests and metadata are written, committed, and pushed. runAppEffects <- do debouncer <- Registry.newDebouncer - let registryEnv pull write = { pull, write, repos: Registry.defaultRepos, workdir: scratchDir, debouncer, cacheRef: registryCacheRef } + repoLocks <- Registry.newRepoLocks + let + registryEnv pull write = + { pull + , write + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptLegacyImporter + } case mode of DryRun -> do token <- Env.lookupRequired Env.githubToken diff --git a/scripts/src/PackageDeleter.purs b/scripts/src/PackageDeleter.purs index 257a7b1a..d93d268b 100644 --- a/scripts/src/PackageDeleter.purs +++ b/scripts/src/PackageDeleter.purs @@ -122,6 +122,7 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -131,6 +132,8 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptPackageDeleter } -- Logging diff --git a/scripts/src/Solver.purs b/scripts/src/Solver.purs index ce615b5a..a61bcacf 100644 --- a/scripts/src/Solver.purs +++ b/scripts/src/Solver.purs @@ -117,7 +117,18 @@ main = launchAff_ do FS.Extra.ensureDirectory cache debouncer <- Registry.newDebouncer - let registryEnv pull write = { pull, write, repos: Registry.defaultRepos, workdir: scratchDir, debouncer, cacheRef: registryCacheRef } + repoLocks <- Registry.newRepoLocks + let + registryEnv pull write = + { pull + , write + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptSolver + } resourceEnv <- Env.lookupResourceEnv token <- Env.lookupRequired Env.githubToken octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl diff --git a/scripts/src/VerifyIntegrity.purs b/scripts/src/VerifyIntegrity.purs index 97aef379..ac79bd7e 100644 --- a/scripts/src/VerifyIntegrity.purs +++ b/scripts/src/VerifyIntegrity.purs @@ -79,6 +79,7 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer + repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -88,6 +89,8 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef + , repoLocks + , process: Registry.ScriptVerifyIntegrity } -- Logging From 8e8de0b34b15e860340384f32fdf7eb2a51acd91 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Fri, 16 Jan 2026 02:55:35 +0200 Subject: [PATCH 05/20] Remove artificial registry repo warmup --- app/src/App/Main.purs | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/app/src/App/Main.purs b/app/src/App/Main.purs index 46774a71..7b6e0e79 100644 --- a/app/src/App/Main.purs +++ b/app/src/App/Main.purs @@ -8,9 +8,7 @@ import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch.Retry as Fetch.Retry import Node.Process as Process -import Registry.App.Effect.Log as Log -import Registry.App.Effect.Registry as Registry -import Registry.App.Server.Env (createServerEnv, runEffects) +import Registry.App.Server.Env (createServerEnv) import Registry.App.Server.JobExecutor as JobExecutor import Registry.App.Server.Router as Router import Registry.App.Server.Scheduler as Scheduler @@ -21,20 +19,13 @@ main = Aff.launchAff_ do Left error -> liftEffect do Console.log $ "Failed to start server: " <> Aff.message error Process.exit' 1 - Right env -> do - -- Initialize registry repo before launching parallel processes, to avoid - -- race condition where both Scheduler and Job Executor try to clone the - -- Registry at the same time - void $ runEffects env do - Log.info "Initializing registry repo..." - Registry.readAllMetadata - liftEffect do - case env.vars.resourceEnv.healthchecksUrl of - Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" - Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl - Aff.launchAff_ $ withRetryLoop "Scheduler" $ Scheduler.runScheduler env - Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor env - Router.runRouter env + Right env -> liftEffect do + case env.vars.resourceEnv.healthchecksUrl of + Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" + Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl + Aff.launchAff_ $ withRetryLoop "Scheduler" $ Scheduler.runScheduler env + Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor env + Router.runRouter env where healthcheck :: String -> Aff Unit healthcheck healthchecksUrl = loop limit From 525edbd93c1938fe9b0a6560a9ab084cc416b671 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Fri, 16 Jan 2026 03:26:20 +0200 Subject: [PATCH 06/20] Pick the right process name when launching the fibers --- app/src/App/Main.purs | 5 +++-- app/src/App/Server/Env.purs | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/src/App/Main.purs b/app/src/App/Main.purs index 7b6e0e79..efacbbf3 100644 --- a/app/src/App/Main.purs +++ b/app/src/App/Main.purs @@ -8,6 +8,7 @@ import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch.Retry as Fetch.Retry import Node.Process as Process +import Registry.App.Effect.Registry as Registry import Registry.App.Server.Env (createServerEnv) import Registry.App.Server.JobExecutor as JobExecutor import Registry.App.Server.Router as Router @@ -23,8 +24,8 @@ main = Aff.launchAff_ do case env.vars.resourceEnv.healthchecksUrl of Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl - Aff.launchAff_ $ withRetryLoop "Scheduler" $ Scheduler.runScheduler env - Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor env + Aff.launchAff_ $ withRetryLoop "Scheduler" $ Scheduler.runScheduler (env { process = Registry.Scheduler }) + Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor (env { process = Registry.JobExecutor }) Router.runRouter env where healthcheck :: String -> Aff Unit diff --git a/app/src/App/Server/Env.purs b/app/src/App/Server/Env.purs index 89fb0b8a..53b71d4d 100644 --- a/app/src/App/Server/Env.purs +++ b/app/src/App/Server/Env.purs @@ -28,7 +28,7 @@ import Registry.App.Effect.PackageSets (PACKAGE_SETS) import Registry.App.Effect.PackageSets as PackageSets import Registry.App.Effect.Pursuit (PURSUIT) import Registry.App.Effect.Pursuit as Pursuit -import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry (Process, REGISTRY) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (SOURCE) import Registry.App.Effect.Source as Source @@ -77,6 +77,7 @@ type ServerEnv = , vars :: ServerEnvVars , debouncer :: Registry.Debouncer , repoLocks :: Registry.RepoLocks + , process :: Process , db :: SQLite , jobId :: Maybe JobId } @@ -122,6 +123,7 @@ createServerEnv = do , vars , octokit , db + , process: Registry.API , jobId: Nothing } @@ -163,7 +165,7 @@ runEffects env operation = Aff.attempt do , debouncer: env.debouncer , cacheRef: env.registryCacheRef , repoLocks: env.repoLocks - , process: Registry.API + , process: env.process } ) # Archive.interpret Archive.handle From 78e62f90ed59ddf65dc8891ad0ad80c42d950104 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sat, 17 Jan 2026 00:00:50 +0200 Subject: [PATCH 07/20] WIP --- app/test/App/API.purs | 9 +++++++-- nix/test/config.nix | 46 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/app/test/App/API.purs b/app/test/App/API.purs index 28f17f90..4413a32d 100644 --- a/app/test/App/API.purs +++ b/app/test/App/API.purs @@ -142,7 +142,7 @@ spec = do Nothing -> Except.throw $ "Expected " <> formatPackageVersion name version <> " to be in metadata." Just published -> do let many' = NonEmptyArray.toArray published.compilers - let expected = map Utils.unsafeVersion [ "0.15.10", "0.15.11" ] + let expected = map Utils.unsafeVersion [ "0.15.10" ] unless (many' == expected) do Except.throw $ "Expected " <> formatPackageVersion name version <> " to have a compiler matrix of " <> Utils.unsafeStringify (map Version.print expected) <> " but got " <> Utils.unsafeStringify (map Version.print many') @@ -191,7 +191,7 @@ spec = do Nothing -> Except.throw $ "Expected " <> formatPackageVersion transitive.name transitive.version <> " to be in metadata." Just published -> do let many' = NonEmptyArray.toArray published.compilers - let expected = map Utils.unsafeVersion [ "0.15.10", "0.15.11" ] + let expected = map Utils.unsafeVersion [ "0.15.10" ] unless (many' == expected) do Except.throw $ "Expected " <> formatPackageVersion transitive.name transitive.version <> " to have a compiler matrix of " <> Utils.unsafeStringify (map Version.print expected) <> " but got " <> Utils.unsafeStringify (map Version.print many') @@ -255,6 +255,11 @@ spec = do -- it from scratch and will fail if effect-4.0.0 is already in storage. We have it in storage -- for the separate integration tests. FS.Extra.remove $ Path.concat [ testFixtures, "registry-storage", "effect-4.0.0.tar.gz" ] + -- Similarly, we remove type-equality files since the unit test publishes it from scratch + -- and will fail if type-equality already has metadata or storage. We have these files for + -- the separate integration tests (scheduler transfer tests). + FS.Extra.remove $ Path.concat [ testFixtures, "registry", "metadata", "type-equality.json" ] + FS.Extra.remove $ Path.concat [ testFixtures, "registry-storage", "type-equality-4.0.1.tar.gz" ] let readFixtures = do diff --git a/nix/test/config.nix b/nix/test/config.nix index afd6f187..83e07a1f 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -288,9 +288,10 @@ let ]; }; } - # Tags for type-equality package (used by two scheduler tests): + # Tags for type-equality package (used by scheduler tests): # 1. Transfer detection: metadata says old-owner, commit URLs point to purescript - # 2. Legacy imports: v4.0.2 is a new version not yet published + # 2. Daily publish: v4.0.2 is a new version not yet published + # We need two mappings: one for old-owner (pre-transfer) and one for purescript (post-transfer) { request = { method = "GET"; @@ -312,7 +313,34 @@ let name = "v4.0.2"; commit = { sha = "type-eq-sha-402"; - # New version not yet published - scheduler detects for legacy import + # New version not yet published - scheduler detects for daily publish + url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-402"; + }; + } + ]; + }; + } + # Tags for type-equality after transfer (when scheduler checks for new versions) + { + request = { + method = "GET"; + url = "/repos/purescript/purescript-type-equality/tags"; + }; + response = { + status = 200; + headers."Content-Type" = "application/json"; + jsonBody = [ + { + name = "v4.0.1"; + commit = { + sha = "type-eq-sha-401"; + url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-401"; + }; + } + { + name = "v4.0.2"; + commit = { + sha = "type-eq-sha-402"; url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-402"; }; } @@ -954,6 +982,9 @@ let gitbot -C "$FIXTURES_DIR/purescript/purescript-console" tag -m "v6.1.0" v6.1.0 gitbot -C "$FIXTURES_DIR/purescript/purescript-unsafe-coerce" tag -m "v6.0.0" v6.0.0 gitbot -C "$FIXTURES_DIR/purescript/purescript-type-equality" tag -m "v4.0.1" v4.0.1 + # Create a new commit for v4.0.2 so it's on a different commit than v4.0.1 + # (the registry rejects publishing when multiple version tags point to the same commit) + gitbot -C "$FIXTURES_DIR/purescript/purescript-type-equality" commit --allow-empty -m "v4.0.2 release" gitbot -C "$FIXTURES_DIR/purescript/purescript-type-equality" tag -m "v4.0.2" v4.0.2 ''; }; @@ -1019,11 +1050,10 @@ let mkdir -p "$STATE_DIR/db" - # Set up git fixtures if needed - if [ ! -d "$REPO_FIXTURES_DIR/purescript" ]; then - echo "Setting up git fixtures..." - ${setupGitFixtures}/bin/setup-git-fixtures "$REPO_FIXTURES_DIR" - fi + # Always recreate git fixtures to ensure clean state + # (the setupGitFixtures script handles cleanup internally) + echo "Setting up git fixtures..." + ${setupGitFixtures}/bin/setup-git-fixtures "$REPO_FIXTURES_DIR" # Run database migrations echo "Running database migrations..." From 02e5cdcd07997ed155742567d7a7b6d0fe0b3bbb Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Fri, 16 Jan 2026 16:13:08 -0500 Subject: [PATCH 08/20] fix ci tests --- app-e2e/src/Test/E2E/Endpoint/Startup.purs | 6 +-- .../registry/metadata/type-equality.json | 2 +- app/src/App/Server/Scheduler.purs | 14 +++--- app/test/App/API.purs | 3 ++ nix/test/config.nix | 49 +++++-------------- 5 files changed, 26 insertions(+), 48 deletions(-) diff --git a/app-e2e/src/Test/E2E/Endpoint/Startup.purs b/app-e2e/src/Test/E2E/Endpoint/Startup.purs index eb17e02c..27aa2d30 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Startup.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Startup.purs @@ -83,7 +83,7 @@ spec = do Spec.describe "scheduleTransfers" do Spec.it "enqueues transfer jobs when package location changes" do - -- type-equality metadata says old-owner, but tags point to purescript + -- type-equality metadata says purescript, but tags point to new-owner jobs <- Client.getJobs let isTypeEqualityTransferJob :: Job -> Boolean @@ -101,8 +101,8 @@ spec = do Transfer { newLocation } -> case newLocation of GitHub { owner } -> - when (owner /= "purescript") do - Assert.fail $ "Expected owner 'purescript' but got '" <> owner <> "'" + when (owner /= "new-owner") do + Assert.fail $ "Expected owner 'new-owner' but got '" <> owner <> "'" _ -> Assert.fail "Expected GitHub location" _ -> Assert.fail "Expected Transfer payload" Just _ -> Assert.fail "Expected TransferJob but got different job type" diff --git a/app/fixtures/registry/metadata/type-equality.json b/app/fixtures/registry/metadata/type-equality.json index 35c13b75..e51b5261 100644 --- a/app/fixtures/registry/metadata/type-equality.json +++ b/app/fixtures/registry/metadata/type-equality.json @@ -1,6 +1,6 @@ { "location": { - "githubOwner": "old-owner", + "githubOwner": "purescript", "githubRepo": "purescript-type-equality" }, "published": { diff --git a/app/src/App/Server/Scheduler.purs b/app/src/App/Server/Scheduler.purs index 1d1b2d5d..f0c88424 100644 --- a/app/src/App/Server/Scheduler.purs +++ b/app/src/App/Server/Scheduler.purs @@ -257,12 +257,14 @@ enqueuePublishJob allMetadata name (Metadata metadata) version ref = do -- Find the highest published version of each dependency within its range let depVersions :: Map PackageName Version - depVersions = Map.mapMaybeWithKey (\depName range -> - case Map.lookup depName allMetadata of - Just (Metadata depMeta) -> - Array.last $ Array.filter (Range.includes range) $ Array.sort $ Array.fromFoldable $ Map.keys depMeta.published - Nothing -> Nothing - ) manifest.dependencies + depVersions = Map.mapMaybeWithKey + ( \depName range -> + case Map.lookup depName allMetadata of + Just (Metadata depMeta) -> + Array.last $ Array.filter (Range.includes range) $ Array.sort $ Array.fromFoldable $ Map.keys depMeta.published + Nothing -> Nothing + ) + manifest.dependencies case compatibleCompilers allMetadata depVersions of Just compilerSet -> pure $ NonEmptySet.min compilerSet diff --git a/app/test/App/API.purs b/app/test/App/API.purs index 4413a32d..78c02e3e 100644 --- a/app/test/App/API.purs +++ b/app/test/App/API.purs @@ -142,6 +142,8 @@ spec = do Nothing -> Except.throw $ "Expected " <> formatPackageVersion name version <> " to be in metadata." Just published -> do let many' = NonEmptyArray.toArray published.compilers + -- Only 0.15.10 is expected because prelude only has 0.15.10 in metadata, + -- so the solver cannot find a solution for 0.15.11 let expected = map Utils.unsafeVersion [ "0.15.10" ] unless (many' == expected) do Except.throw $ "Expected " <> formatPackageVersion name version <> " to have a compiler matrix of " <> Utils.unsafeStringify (map Version.print expected) <> " but got " <> Utils.unsafeStringify (map Version.print many') @@ -191,6 +193,7 @@ spec = do Nothing -> Except.throw $ "Expected " <> formatPackageVersion transitive.name transitive.version <> " to be in metadata." Just published -> do let many' = NonEmptyArray.toArray published.compilers + -- Only 0.15.10 is expected because prelude only has 0.15.10 in metadata let expected = map Utils.unsafeVersion [ "0.15.10" ] unless (many' == expected) do Except.throw $ "Expected " <> formatPackageVersion transitive.name transitive.version <> " to have a compiler matrix of " <> Utils.unsafeStringify (map Version.print expected) <> " but got " <> Utils.unsafeStringify (map Version.print many') diff --git a/nix/test/config.nix b/nix/test/config.nix index 83e07a1f..e55d1746 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -288,39 +288,9 @@ let ]; }; } - # Tags for type-equality package (used by scheduler tests): - # 1. Transfer detection: metadata says old-owner, commit URLs point to purescript - # 2. Daily publish: v4.0.2 is a new version not yet published - # We need two mappings: one for old-owner (pre-transfer) and one for purescript (post-transfer) - { - request = { - method = "GET"; - url = "/repos/old-owner/purescript-type-equality/tags"; - }; - response = { - status = 200; - headers."Content-Type" = "application/json"; - jsonBody = [ - { - name = "v4.0.1"; - commit = { - sha = "type-eq-sha-401"; - # Points to actual owner - scheduler detects this transfer - url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-401"; - }; - } - { - name = "v4.0.2"; - commit = { - sha = "type-eq-sha-402"; - # New version not yet published - scheduler detects for daily publish - url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-402"; - }; - } - ]; - }; - } - # Tags for type-equality after transfer (when scheduler checks for new versions) + # Tags for type-equality package (used by two scheduler tests): + # 1. Transfer detection: metadata says purescript, commit URLs point to new-owner + # 2. Legacy imports: v4.0.2 is a new version not yet published { request = { method = "GET"; @@ -334,14 +304,16 @@ let name = "v4.0.1"; commit = { sha = "type-eq-sha-401"; - url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-401"; + # Points to new owner - scheduler detects this transfer + url = "https://api.github.com/repos/new-owner/purescript-type-equality/commits/type-eq-sha-401"; }; } { name = "v4.0.2"; commit = { sha = "type-eq-sha-402"; - url = "https://api.github.com/repos/purescript/purescript-type-equality/commits/type-eq-sha-402"; + # New version not yet published - scheduler detects for legacy import + url = "https://api.github.com/repos/new-owner/purescript-type-equality/commits/type-eq-sha-402"; }; } ]; @@ -429,14 +401,15 @@ let # Parse metadata files to get the actual published versions (not just package names) # Returns a set like { "prelude-6.0.1" = true; "type-equality-4.0.1" = true; } - publishedVersions = lib.foldl' (acc: fileName: + publishedVersions = lib.foldl' ( + acc: fileName: let packageName = lib.removeSuffix ".json" fileName; metadata = builtins.fromJSON (builtins.readFile (metadataFixturesDir + "/${fileName}")); - versions = builtins.attrNames (metadata.published or {}); + versions = builtins.attrNames (metadata.published or { }); in acc // lib.genAttrs (map (v: "${packageName}-${v}") versions) (_: true) - ) {} metadataFiles; + ) { } metadataFiles; # ============================================================================ # UNIFIED STORAGE MAPPINGS WITH WIREMOCK SCENARIOS From 4caa9161081c832a96479661b0dfe85a46a350c7 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sat, 17 Jan 2026 15:48:13 +0200 Subject: [PATCH 09/20] Move the scheduler stuff back to separate scripts --- app-e2e/spago.yaml | 2 + app-e2e/src/Test/E2E/Endpoint/Startup.purs | 138 +-------- app-e2e/src/Test/E2E/Scripts.purs | 292 ++++++++++++++++++++ app-e2e/src/Test/Main.purs | 10 +- app/src/App/Effect/Registry.purs | 86 +----- app/src/App/Main.purs | 5 +- app/src/App/Server/Env.purs | 9 +- app/src/App/Server/Scheduler.purs | 307 --------------------- flake.nix | 2 + nix/overlay.nix | 12 + scripts/src/ArchiveSeeder.purs | 3 - scripts/src/CompilerVersions.purs | 3 - scripts/src/DailyImporter.purs | 268 ++++++++++++++++++ scripts/src/LegacyImporter.purs | 3 - scripts/src/PackageDeleter.purs | 3 - scripts/src/PackageSetUpdater.purs | 240 ++++++++++++++++ scripts/src/PackageTransferrer.purs | 235 ++++++++++++++++ scripts/src/Solver.purs | 3 - scripts/src/VerifyIntegrity.purs | 3 - spago.lock | 4 + 20 files changed, 1065 insertions(+), 563 deletions(-) create mode 100644 app-e2e/src/Test/E2E/Scripts.purs delete mode 100644 app/src/App/Server/Scheduler.purs create mode 100644 scripts/src/DailyImporter.purs create mode 100644 scripts/src/PackageSetUpdater.purs create mode 100644 scripts/src/PackageTransferrer.purs diff --git a/app-e2e/spago.yaml b/app-e2e/spago.yaml index fb3804b9..90002234 100644 --- a/app-e2e/spago.yaml +++ b/app-e2e/spago.yaml @@ -21,8 +21,10 @@ package: - registry-app - registry-foreign - registry-lib + - registry-scripts - registry-test-utils - routing-duplex + - run - spec - spec-node - strings diff --git a/app-e2e/src/Test/E2E/Endpoint/Startup.purs b/app-e2e/src/Test/E2E/Endpoint/Startup.purs index 27aa2d30..0b60c2fe 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Startup.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Startup.purs @@ -1,21 +1,15 @@ --- | E2E tests for the Scheduler and JobExecutor startup, covering: --- | - scheduleDailyPublish: Detects new package versions via GitHub tags --- | - scheduleTransfers: Detects packages that moved to new GitHub locations --- | - schedulePackageSetUpdates: Detects recent uploads for package set inclusion +-- | E2E tests for server startup behavior (non-scheduler). +-- | -- | - checkIfNewCompiler: Detects new compiler and enqueues matrix jobs -- | -- | IMPORTANT: These tests must run BEFORE resetTestState is called, since --- | the scheduler runs at server startup and creates jobs that would be cleared. +-- | the jobs are created at server startup and would be cleared. module Test.E2E.Endpoint.Startup (spec) where import Registry.App.Prelude import Data.Array as Array -import Data.Map as Map import Registry.API.V1 (Job(..)) -import Registry.Location (Location(..)) -import Registry.Operation (AuthenticatedPackageOperation(..)) -import Registry.Operation as Operation import Registry.PackageName as PackageName import Registry.Test.Assert as Assert import Registry.Version as Version @@ -25,107 +19,6 @@ import Test.Spec as Spec spec :: E2ESpec spec = do - Spec.describe "scheduleDailyPublish" do - Spec.it "enqueues publish jobs for new package versions discovered via GitHub tags" do - -- The scheduler runs at server startup and should have already - -- fetched tags for packages in the registry metadata. - -- type-equality has v4.0.1 published but v4.0.2 in tags (per wiremock config) - jobs <- Client.getJobs - - -- Find publish jobs for type-equality - let - isTypeEqualityPublishJob :: Job -> Boolean - isTypeEqualityPublishJob = case _ of - PublishJob { packageName, packageVersion } -> - packageName == unsafeFromRight (PackageName.parse "type-equality") - && packageVersion - == unsafeFromRight (Version.parse "4.0.2") - _ -> false - - typeEqualityJob = Array.find isTypeEqualityPublishJob jobs - - case typeEqualityJob of - Just (PublishJob { payload }) -> do - -- The scheduler determines a compatible compiler by looking at the previous - -- version's dependencies and finding the intersection of their supported compilers. - -- type-equality@4.0.1 has no dependencies, so the scheduler falls back to the - -- lowest compiler from the previous version (0.15.10). - let expectedCompiler = unsafeFromRight (Version.parse "0.15.10") - when (payload.compiler /= expectedCompiler) do - Assert.fail $ "Expected compiler 0.15.10 but got " <> Version.print payload.compiler - Just _ -> Assert.fail "Expected PublishJob but got different job type" - Nothing -> do - -- Log what jobs we did find for debugging - let publishJobs = Array.filter isPublishJob jobs - Assert.fail $ "Expected to find a publish job for type-equality@4.0.2 but found " - <> show (Array.length publishJobs) - <> " publish jobs: " - <> show (map formatPublishJob publishJobs) - - Spec.it "does not enqueue jobs for already-published versions" do - jobs <- Client.getJobs - - -- type-equality v4.0.1 is already published, should NOT have a new job - let - isDuplicateJob :: Job -> Boolean - isDuplicateJob = case _ of - PublishJob { packageName, packageVersion } -> - packageName == unsafeFromRight (PackageName.parse "type-equality") - && packageVersion - == unsafeFromRight (Version.parse "4.0.1") - _ -> false - - duplicateJob = Array.find isDuplicateJob jobs - - case duplicateJob of - Nothing -> pure unit -- Good, no duplicate job - Just _ -> Assert.fail "Found unexpected publish job for already-published type-equality@4.0.1" - - Spec.describe "scheduleTransfers" do - Spec.it "enqueues transfer jobs when package location changes" do - -- type-equality metadata says purescript, but tags point to new-owner - jobs <- Client.getJobs - let - isTypeEqualityTransferJob :: Job -> Boolean - isTypeEqualityTransferJob = case _ of - TransferJob { packageName } -> - packageName == unsafeFromRight (PackageName.parse "type-equality") - _ -> false - case Array.find isTypeEqualityTransferJob jobs of - Just (TransferJob { packageName, payload }) -> do - -- Verify packageName - when (packageName /= unsafeFromRight (PackageName.parse "type-equality")) do - Assert.fail $ "Wrong package name: " <> PackageName.print packageName - -- Verify newLocation in payload - case payload.payload of - Transfer { newLocation } -> - case newLocation of - GitHub { owner } -> - when (owner /= "new-owner") do - Assert.fail $ "Expected owner 'new-owner' but got '" <> owner <> "'" - _ -> Assert.fail "Expected GitHub location" - _ -> Assert.fail "Expected Transfer payload" - Just _ -> Assert.fail "Expected TransferJob but got different job type" - Nothing -> do - let transferJobs = Array.filter isTransferJob jobs - Assert.fail $ "Expected to find a transfer job for 'type-equality' but found " - <> show (Array.length transferJobs) - <> " transfer jobs" - - Spec.describe "schedulePackageSetUpdates" do - Spec.it "enqueues package set update for recent uploads not in set" do - jobs <- Client.getJobs - let packageSetJobs = Array.filter isPackageSetJob jobs - case Array.head packageSetJobs of - Just (PackageSetJob { payload }) -> - case payload of - Operation.PackageSetUpdate { packages } -> - case Map.lookup (unsafeFromRight $ PackageName.parse "type-equality") packages of - Just (Just _) -> pure unit - _ -> Assert.fail "Expected type-equality in package set update" - Just _ -> Assert.fail "Expected PackageSetJob but got different job type" - Nothing -> Assert.fail "Expected package set job to be enqueued" - Spec.describe "checkIfNewCompiler" do Spec.it "enqueues matrix jobs for packages with no dependencies when new compiler detected" do -- The test env has compilers 0.15.10 and 0.15.11 available. @@ -158,28 +51,3 @@ spec = do <> show (Array.length matrixJobs) <> " matrix jobs for packages: " <> show (map PackageName.print matrixPackages) - --- | Check if a job is a PublishJob -isPublishJob :: Job -> Boolean -isPublishJob = case _ of - PublishJob _ -> true - _ -> false - --- | Format a PublishJob for debugging output -formatPublishJob :: Job -> String -formatPublishJob = case _ of - PublishJob { packageName, packageVersion } -> - PackageName.print packageName <> "@" <> Version.print packageVersion - _ -> "" - --- | Check if a job is a TransferJob -isTransferJob :: Job -> Boolean -isTransferJob = case _ of - TransferJob _ -> true - _ -> false - --- | Check if a job is a PackageSetJob -isPackageSetJob :: Job -> Boolean -isPackageSetJob = case _ of - PackageSetJob _ -> true - _ -> false diff --git a/app-e2e/src/Test/E2E/Scripts.purs b/app-e2e/src/Test/E2E/Scripts.purs new file mode 100644 index 00000000..e4b0ef9f --- /dev/null +++ b/app-e2e/src/Test/E2E/Scripts.purs @@ -0,0 +1,292 @@ +-- | End-to-end tests for the cronjob scripts: +-- | - DailyImporter: Detects new package versions via GitHub tags +-- | - PackageSetUpdater: Detects recent uploads for package set inclusion +-- | - PackageTransferrer: Detects packages that moved to new GitHub locations +-- | +-- | These tests verify that the scripts properly enqueue jobs via the API. +module Test.E2E.Scripts (spec) where + +import Registry.App.Prelude + +import Control.Monad.Reader (ask) +import Data.Array as Array +import Data.Map as Map +import Effect.Aff as Aff +import Node.Path as Path +import Node.Process as Process +import Registry.API.V1 (Job(..)) +import Registry.App.CLI.Git as Git +import Registry.App.Effect.Cache as Cache +import Registry.App.Effect.Env as Env +import Registry.App.Effect.GitHub as GitHub +import Registry.App.Effect.Log as Log +import Registry.App.Effect.Registry as Registry +import Registry.Foreign.Octokit as Octokit +import Registry.Location (Location(..)) +import Registry.Operation (AuthenticatedPackageOperation(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Registry.Scripts.DailyImporter as DailyImporter +import Registry.Scripts.PackageSetUpdater as PackageSetUpdater +import Registry.Scripts.PackageTransferrer as PackageTransferrer +import Registry.Test.Assert as Assert +import Registry.Version as Version +import Run as Run +import Run.Except as Except +import Test.E2E.Support.Client as Client +import Test.E2E.Support.Env (E2ESpec, E2E) +import Test.Spec as Spec + +spec :: E2ESpec +spec = do + Spec.describe "DailyImporter" do + Spec.it "enqueues publish jobs for new package versions discovered via GitHub tags" do + runDailyImporterScript + jobs <- Client.getJobs + + -- type-equality has v4.0.1 published but v4.0.2 in tags (per wiremock config) + let + isTypeEqualityPublishJob :: Job -> Boolean + isTypeEqualityPublishJob = case _ of + PublishJob { packageName, packageVersion } -> + packageName == unsafeFromRight (PackageName.parse "type-equality") + && packageVersion + == unsafeFromRight (Version.parse "4.0.2") + _ -> false + + typeEqualityJob = Array.find isTypeEqualityPublishJob jobs + + case typeEqualityJob of + Just (PublishJob { payload }) -> do + -- Verify compiler selection logic + let expectedCompiler = unsafeFromRight (Version.parse "0.15.10") + when (payload.compiler /= expectedCompiler) do + Assert.fail $ "Expected compiler 0.15.10 but got " <> Version.print payload.compiler + Just _ -> Assert.fail "Expected PublishJob but got different job type" + Nothing -> do + let publishJobs = Array.filter isPublishJob jobs + Assert.fail $ "Expected to find a publish job for type-equality@4.0.2 but found " + <> show (Array.length publishJobs) + <> " publish jobs: " + <> show (map formatPublishJob publishJobs) + + Spec.it "does not enqueue jobs for already-published versions" do + runDailyImporterScript + jobs <- Client.getJobs + + -- type-equality v4.0.1 is already published, should NOT have a new job + let + isDuplicateJob :: Job -> Boolean + isDuplicateJob = case _ of + PublishJob { packageName, packageVersion } -> + packageName == unsafeFromRight (PackageName.parse "type-equality") + && packageVersion + == unsafeFromRight (Version.parse "4.0.1") + _ -> false + + duplicateJob = Array.find isDuplicateJob jobs + + case duplicateJob of + Nothing -> pure unit -- Good, no duplicate job + Just _ -> Assert.fail "Found unexpected publish job for already-published type-equality@4.0.1" + + Spec.describe "PackageTransferrer" do + Spec.it "enqueues transfer jobs when package location changes" do + runPackageTransferrerScript + -- type-equality metadata says purescript, but tags point to new-owner + jobs <- Client.getJobs + let + isTypeEqualityTransferJob :: Job -> Boolean + isTypeEqualityTransferJob = case _ of + TransferJob { packageName } -> + packageName == unsafeFromRight (PackageName.parse "type-equality") + _ -> false + case Array.find isTypeEqualityTransferJob jobs of + Just (TransferJob { packageName, payload }) -> do + -- Verify packageName + when (packageName /= unsafeFromRight (PackageName.parse "type-equality")) do + Assert.fail $ "Wrong package name: " <> PackageName.print packageName + -- Verify newLocation in payload + case payload.payload of + Transfer { newLocation } -> + case newLocation of + GitHub { owner } -> + when (owner /= "new-owner") do + Assert.fail $ "Expected owner 'new-owner' but got '" <> owner <> "'" + _ -> Assert.fail "Expected GitHub location" + _ -> Assert.fail "Expected Transfer payload" + Just _ -> Assert.fail "Expected TransferJob but got different job type" + Nothing -> do + let transferJobs = Array.filter isTransferJob jobs + Assert.fail $ "Expected to find a transfer job for 'type-equality' but found " + <> show (Array.length transferJobs) + <> " transfer jobs" + + Spec.describe "PackageSetUpdater" do + Spec.it "enqueues package set update for recent uploads not in set" do + runPackageSetUpdaterScript + jobs <- Client.getJobs + let packageSetJobs = Array.filter isPackageSetJob jobs + case Array.head packageSetJobs of + Just (PackageSetJob { payload }) -> + case payload of + Operation.PackageSetUpdate { packages } -> + case Map.lookup (unsafeFromRight $ PackageName.parse "type-equality") packages of + Just (Just _) -> pure unit + _ -> Assert.fail "Expected type-equality in package set update" + Just _ -> Assert.fail "Expected PackageSetJob but got different job type" + Nothing -> Assert.fail "Expected package set job to be enqueued" + +-- | Run the DailyImporter script in Submit mode +runDailyImporterScript :: E2E Unit +runDailyImporterScript = do + { stateDir } <- ask + + -- Set up environment + liftEffect $ Process.chdir stateDir + + -- Get resource env from environment variables + resourceEnv <- liftEffect Env.lookupResourceEnv + token <- liftEffect $ Env.lookupRequired Env.githubToken + + githubCacheRef <- liftAff Cache.newCacheRef + registryCacheRef <- liftAff Cache.newCacheRef + let cache = Path.concat [ stateDir, "scratch", ".cache" ] + + octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl + debouncer <- liftAff Registry.newDebouncer + + let + registryEnv :: Registry.RegistryEnv + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: Path.concat [ stateDir, "scratch" ] + , debouncer + , cacheRef: registryCacheRef + } + + result <- liftAff $ + DailyImporter.runDailyImport DailyImporter.Submit resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Quiet) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' + + case result of + Left err -> liftAff $ Aff.throwError $ Aff.error $ "DailyImporter failed: " <> err + Right _ -> pure unit + +-- | Run the PackageTransferrer script in Submit mode +runPackageTransferrerScript :: E2E Unit +runPackageTransferrerScript = do + { stateDir, privateKey } <- ask + + -- Set up environment + liftEffect $ Process.chdir stateDir + + -- Get resource env from environment variables + resourceEnv <- liftEffect Env.lookupResourceEnv + token <- liftEffect $ Env.lookupRequired Env.githubToken + + githubCacheRef <- liftAff Cache.newCacheRef + registryCacheRef <- liftAff Cache.newCacheRef + let cache = Path.concat [ stateDir, "scratch", ".cache" ] + + octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl + debouncer <- liftAff Registry.newDebouncer + + let + registryEnv :: Registry.RegistryEnv + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: Path.concat [ stateDir, "scratch" ] + , debouncer + , cacheRef: registryCacheRef + } + + result <- liftAff $ + PackageTransferrer.runPackageTransferrer PackageTransferrer.Submit (Just privateKey) resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Quiet) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' + + case result of + Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageTransferrer failed: " <> err + Right _ -> pure unit + +-- | Run the PackageSetUpdater script in Submit mode +runPackageSetUpdaterScript :: E2E Unit +runPackageSetUpdaterScript = do + { stateDir, privateKey } <- ask + + -- Set up environment + liftEffect $ Process.chdir stateDir + + -- Get resource env from environment variables + resourceEnv <- liftEffect Env.lookupResourceEnv + token <- liftEffect $ Env.lookupRequired Env.githubToken + + githubCacheRef <- liftAff Cache.newCacheRef + registryCacheRef <- liftAff Cache.newCacheRef + let cache = Path.concat [ stateDir, "scratch", ".cache" ] + + octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl + debouncer <- liftAff Registry.newDebouncer + + let + registryEnv :: Registry.RegistryEnv + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: Path.concat [ stateDir, "scratch" ] + , debouncer + , cacheRef: registryCacheRef + } + + result <- liftAff $ + PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit (Just privateKey) resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Quiet) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' + + case result of + Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageSetUpdater failed: " <> err + Right _ -> pure unit + +-- | Check if a job is a PublishJob +isPublishJob :: Job -> Boolean +isPublishJob = case _ of + PublishJob _ -> true + _ -> false + +-- | Format a PublishJob for debugging output +formatPublishJob :: Job -> String +formatPublishJob = case _ of + PublishJob { packageName, packageVersion } -> + PackageName.print packageName <> "@" <> Version.print packageVersion + _ -> "" + +-- | Check if a job is a TransferJob +isTransferJob :: Job -> Boolean +isTransferJob = case _ of + TransferJob _ -> true + _ -> false + +-- | Check if a job is a PackageSetJob +isPackageSetJob :: Job -> Boolean +isPackageSetJob = case _ of + PackageSetJob _ -> true + _ -> false diff --git a/app-e2e/src/Test/Main.purs b/app-e2e/src/Test/Main.purs index 5a81fbfd..a0f88464 100644 --- a/app-e2e/src/Test/Main.purs +++ b/app-e2e/src/Test/Main.purs @@ -10,6 +10,7 @@ import Test.E2E.Endpoint.Startup as Startup import Test.E2E.Endpoint.Transfer as Transfer import Test.E2E.Endpoint.Unpublish as Unpublish import Test.E2E.GitHubIssue as GitHubIssue +import Test.E2E.Scripts as Scripts import Test.E2E.Support.Env (assertReposClean, mkTestEnv, resetTestState, runE2E, stashGitFixtures, waitForAllPendingJobs) import Test.E2E.Workflow as Workflow import Test.Spec (hoistSpec) @@ -22,14 +23,10 @@ main :: Effect Unit main = do env <- mkTestEnv runSpecAndExitProcess' config [ consoleReporter ] $ hoistE2E env do - -- The scheduler runs at startup and enqueues a bunch of jobs in the DB, - -- so we need to run these tests without cleaning out the state first + -- Run startup tests FIRST, before jobs are processed Spec.describe "Startup" Startup.spec - -- After scheduler tests, wait for startup jobs to complete and stash the - -- git fixtures state. This ensures that subsequent tests can reset to - -- a state where startup jobs (like new compiler matrix jobs) have already - -- updated the metadata. + -- Then wait for pending jobs and stash the git fixtures Spec.describe "Setup" do Spec.it "waits for startup jobs and stashes fixtures" do waitForAllPendingJobs @@ -46,6 +43,7 @@ main = do Spec.describe "Workflows" do Spec.describe "GitHubIssue" GitHubIssue.spec Spec.describe "Multi-operation" Workflow.spec + Spec.describe "Scripts" Scripts.spec where hoistE2E env = hoistSpec identity (\_ m -> runE2E env m) config = diff --git a/app/src/App/Effect/Registry.purs b/app/src/App/Effect/Registry.purs index 4d34f51e..e9b123fb 100644 --- a/app/src/App/Effect/Registry.purs +++ b/app/src/App/Effect/Registry.purs @@ -18,8 +18,6 @@ import Data.Set as Set import Data.String as String import Data.Time.Duration as Duration import Effect.Aff as Aff -import Effect.Aff.AVar (AVar) -import Effect.Aff.AVar as AVar import Effect.Ref as Ref import JSON as JSON import Node.FS.Aff as FS.Aff @@ -161,85 +159,6 @@ data RepoKey derive instance Eq RepoKey derive instance Ord RepoKey --- | Identifies which process is using the registry, for lock ownership tracking. -data Process - = Scheduler - | JobExecutor - | API - | ScriptLegacyImporter - | ScriptPackageDeleter - | ScriptSolver - | ScriptVerifyIntegrity - | ScriptCompilerVersions - | ScriptArchiveSeeder - -derive instance Eq Process - -instance Show Process where - show Scheduler = "Scheduler" - show JobExecutor = "JobExecutor" - show API = "API" - show ScriptLegacyImporter = "ScriptLegacyImporter" - show ScriptPackageDeleter = "ScriptPackageDeleter" - show ScriptSolver = "ScriptSolver" - show ScriptVerifyIntegrity = "ScriptVerifyIntegrity" - show ScriptCompilerVersions = "ScriptCompilerVersions" - show ScriptArchiveSeeder = "ScriptArchiveSeeder" - --- | A lock for a single repository, tracking both the mutex and the owner. -type RepoLock = { lock :: AVar Unit, owner :: Ref (Maybe Process) } - --- | Per-repository locks to prevent concurrent access. -type RepoLocks = Ref (Map RepoKey RepoLock) - --- | Create a new empty set of repo locks. -newRepoLocks :: forall m. MonadEffect m => m RepoLocks -newRepoLocks = liftEffect $ Ref.new Map.empty - --- | Get or create a lock for a repository. -getOrCreateLock :: RepoLocks -> RepoKey -> Aff RepoLock -getOrCreateLock locksRef key = do - locks <- liftEffect $ Ref.read locksRef - case Map.lookup key locks of - Just lock -> pure lock - Nothing -> do - lock <- AVar.new unit - owner <- liftEffect $ Ref.new Nothing - let repoLock = { lock, owner } - liftEffect $ Ref.modify_ (Map.insert key repoLock) locksRef - pure repoLock - --- | Acquire a repository lock, run an action, and release the lock. --- | The lock prevents concurrent access to the same repository. -withRepoLock - :: forall r a - . Process - -> RepoLocks - -> RepoKey - -> Run (LOG + AFF + EFFECT + r) a - -> Run (LOG + AFF + EFFECT + r) a -withRepoLock process locks key action = do - repoLock <- Run.liftAff $ getOrCreateLock locks key - Run.liftAff $ AVar.take repoLock.lock - Run.liftEffect $ Ref.write (Just process) repoLock.owner - result <- action - Run.liftEffect $ Ref.write Nothing repoLock.owner - Run.liftAff $ AVar.put unit repoLock.lock - pure result - --- | Clear any locks owned by a specific process. --- | Used to clean up orphaned locks when a process crashes and restarts. -clearOwnLocks :: forall r. Process -> RepoLocks -> Run (LOG + AFF + EFFECT + r) Unit -clearOwnLocks process locksRef = do - locks <- Run.liftEffect $ Ref.read locksRef - for_ (Map.toUnfoldable locks :: Array _) \(Tuple _ repoLock) -> do - owner <- Run.liftEffect $ Ref.read repoLock.owner - when (owner == Just process) do - Log.warn $ "Clearing orphaned lock for " <> show process - Run.liftEffect $ Ref.write Nothing repoLock.owner - -- Put the unit back to release the lock - Run.liftAff $ AVar.put unit repoLock.lock - -- | Validate that a repository is in a valid state. -- | If the repo is corrupted (e.g., from an interrupted clone), delete it. validateRepo :: forall r. FilePath -> Run (LOG + AFF + EFFECT + r) Unit @@ -307,8 +226,6 @@ type RegistryEnv = , write :: WriteMode , debouncer :: Debouncer , cacheRef :: CacheRef - , repoLocks :: RepoLocks - , process :: Process } type Debouncer = Ref (Map FilePath DateTime) @@ -807,9 +724,8 @@ handle env = Cache.interpret _registryCache (Cache.handleMemory env.cacheRef) << -- | Get the repository at the given key, recording whether the pull or clone -- | had any effect (ie. if the repo was already up-to-date). - -- | Uses per-repository locking to prevent race conditions during clone. pull :: RepoKey -> Run _ (Either String GitResult) - pull repoKey = withRepoLock env.process env.repoLocks repoKey do + pull repoKey = do let path = repoPath repoKey address = repoAddress repoKey diff --git a/app/src/App/Main.purs b/app/src/App/Main.purs index efacbbf3..22c873f5 100644 --- a/app/src/App/Main.purs +++ b/app/src/App/Main.purs @@ -8,11 +8,9 @@ import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch.Retry as Fetch.Retry import Node.Process as Process -import Registry.App.Effect.Registry as Registry import Registry.App.Server.Env (createServerEnv) import Registry.App.Server.JobExecutor as JobExecutor import Registry.App.Server.Router as Router -import Registry.App.Server.Scheduler as Scheduler main :: Effect Unit main = Aff.launchAff_ do @@ -24,8 +22,7 @@ main = Aff.launchAff_ do case env.vars.resourceEnv.healthchecksUrl of Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl - Aff.launchAff_ $ withRetryLoop "Scheduler" $ Scheduler.runScheduler (env { process = Registry.Scheduler }) - Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor (env { process = Registry.JobExecutor }) + Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor env Router.runRouter env where healthcheck :: String -> Aff Unit diff --git a/app/src/App/Server/Env.purs b/app/src/App/Server/Env.purs index 53b71d4d..70e5698f 100644 --- a/app/src/App/Server/Env.purs +++ b/app/src/App/Server/Env.purs @@ -28,7 +28,7 @@ import Registry.App.Effect.PackageSets (PACKAGE_SETS) import Registry.App.Effect.PackageSets as PackageSets import Registry.App.Effect.Pursuit (PURSUIT) import Registry.App.Effect.Pursuit as Pursuit -import Registry.App.Effect.Registry (Process, REGISTRY) +import Registry.App.Effect.Registry (REGISTRY) import Registry.App.Effect.Registry as Registry import Registry.App.Effect.Source (SOURCE) import Registry.App.Effect.Source as Source @@ -76,8 +76,6 @@ type ServerEnv = , octokit :: Octokit , vars :: ServerEnvVars , debouncer :: Registry.Debouncer - , repoLocks :: Registry.RepoLocks - , process :: Process , db :: SQLite , jobId :: Maybe JobId } @@ -96,7 +94,6 @@ createServerEnv = do octokit <- Octokit.newOctokit vars.token vars.resourceEnv.githubApiUrl debouncer <- Registry.newDebouncer - repoLocks <- Registry.newRepoLocks db <- liftEffect $ SQLite.connect { database: vars.resourceEnv.databaseUrl.path @@ -114,7 +111,6 @@ createServerEnv = do pure { debouncer - , repoLocks , githubCacheRef , legacyCacheRef , registryCacheRef @@ -123,7 +119,6 @@ createServerEnv = do , vars , octokit , db - , process: Registry.API , jobId: Nothing } @@ -164,8 +159,6 @@ runEffects env operation = Aff.attempt do , workdir: scratchDir , debouncer: env.debouncer , cacheRef: env.registryCacheRef - , repoLocks: env.repoLocks - , process: env.process } ) # Archive.interpret Archive.handle diff --git a/app/src/App/Server/Scheduler.purs b/app/src/App/Server/Scheduler.purs deleted file mode 100644 index f0c88424..00000000 --- a/app/src/App/Server/Scheduler.purs +++ /dev/null @@ -1,307 +0,0 @@ -module Registry.App.Server.Scheduler - ( runScheduler - ) where - -import Registry.App.Prelude - -import Data.Array as Array -import Data.Array.NonEmpty as NonEmptyArray -import Data.DateTime as DateTime -import Data.Map as Map -import Data.Set as Set -import Data.Set.NonEmpty (NonEmptySet) -import Data.Set.NonEmpty as NonEmptySet -import Data.String as String -import Data.Time.Duration (Hours(..)) -import Effect.Aff (Milliseconds(..)) -import Effect.Aff as Aff -import Registry.App.Auth as Auth -import Registry.App.CLI.PursVersions as PursVersions -import Registry.App.Effect.Db as Db -import Registry.App.Effect.Env as Env -import Registry.App.Effect.GitHub as GitHub -import Registry.App.Effect.Log as Log -import Registry.App.Effect.PackageSets as PackageSets -import Registry.App.Effect.Registry as Registry -import Registry.App.Legacy.LenientVersion as LenientVersion -import Registry.App.Server.Env (ServerEffects, ServerEnv, runEffects) -import Registry.Foreign.Octokit as Octokit -import Registry.Location (Location(..)) -import Registry.Operation as Operation -import Registry.PackageName as PackageName -import Registry.PackageSet (PackageSet(..)) -import Registry.Range as Range -import Run (Run) - --- | The scheduler loop runs immediately, then every 24 hours. --- | It checks for work that needs to be enqueued (transfers, package set --- | updates, legacy imports) and creates the appropriate jobs. -runScheduler :: ServerEnv -> Aff (Either Aff.Error Unit) -runScheduler env = runEffects env do - Log.info "Starting Scheduler" - loop - where - sleepTime = Milliseconds (1000.0 * 60.0 * 60.0 * 24.0) - - loop = do - -- Run all scheduling checks - scheduleTransfers - schedulePackageSetUpdates - scheduleDailyPublish - Log.info "Scheduler cycle complete, sleeping for 24 hours..." - -- Sleep for a while, then run again - liftAff $ Aff.delay sleepTime - loop - --- | Check for packages that have moved and enqueue transfer jobs. -scheduleTransfers :: Run ServerEffects Unit -scheduleTransfers = do - Log.info "Scheduler: checking for package transfers..." - allMetadata <- Registry.readAllMetadata - - -- Check each package for location changes - transfersNeeded <- Array.catMaybes <$> for (Map.toUnfoldable allMetadata) \(Tuple name (Metadata metadata)) -> - case metadata.location of - Git _ -> pure Nothing -- Skip non-GitHub packages - GitHub registered -> do - -- Fetch tags to see if repo has moved - GitHub.listTags { owner: registered.owner, repo: registered.repo } >>= case _ of - Left _ -> pure Nothing -- Can't fetch tags, skip - Right tags | Array.null tags -> pure Nothing -- No tags, skip - Right tags -> case Array.head tags of - Nothing -> pure Nothing - Just tag -> - -- Parse the tag URL to get actual current location - case tagUrlToRepoUrl tag.url of - Nothing -> pure Nothing - Just actual - | locationsMatch registered actual -> pure Nothing -- No change - | otherwise -> pure $ Just { name, newLocation: GitHub { owner: actual.owner, repo: actual.repo, subdir: registered.subdir } } - - case Array.length transfersNeeded of - 0 -> Log.info "No packages require transferring." - n -> do - Log.info $ show n <> " packages need transferring" - for_ transfersNeeded \{ name, newLocation } -> - enqueueTransferJob name newLocation - --- | Parse GitHub API tag URL to extract owner/repo --- | Example: https://api.github.com/repos/octocat/Hello-World/commits/abc123 -tagUrlToRepoUrl :: String -> Maybe { owner :: String, repo :: String } -tagUrlToRepoUrl url = do - noPrefix <- String.stripPrefix (String.Pattern "https://api.github.com/repos/") url - case Array.take 2 $ String.split (String.Pattern "/") noPrefix of - [ owner, repo ] -> Just { owner, repo: String.toLower repo } - _ -> Nothing - --- | Case-insensitive comparison of GitHub locations -locationsMatch :: forall r. { owner :: String, repo :: String | r } -> { owner :: String, repo :: String } -> Boolean -locationsMatch loc1 loc2 = - String.toLower loc1.owner == String.toLower loc2.owner - && String.toLower loc1.repo - == String.toLower loc2.repo - -enqueueTransferJob :: PackageName -> Location -> Run ServerEffects Unit -enqueueTransferJob name newLocation = do - -- Check if transfer job already exists - existingJob <- Db.selectTransferJob name - case existingJob of - Just _ -> Log.debug $ "Transfer job already exists for " <> PackageName.print name - Nothing -> do - let payload = { name, newLocation } - let rawPayload = stringifyJson Operation.transferCodec payload - { privateKey } <- Env.askPacchettiBotti - case Auth.signPayload { privateKey, rawPayload } of - Left _ -> Log.error $ "Failed to sign transfer for " <> PackageName.print name - Right signature -> do - jobId <- Db.insertTransferJob { payload, rawPayload, signature } - Log.info $ "Enqueued transfer job " <> unwrap jobId <> " for " <> PackageName.print name - --- | Check for recent uploads and enqueue package set update job. -schedulePackageSetUpdates :: Run ServerEffects Unit -schedulePackageSetUpdates = do - Log.info "Scheduler: checking for package set updates..." - - -- Get the current package set - latestPackageSet <- Registry.readLatestPackageSet >>= case _ of - Nothing -> do - Log.warn "No package set found, skipping package set updates" - pure Nothing - Just set -> pure (Just set) - - for_ latestPackageSet \packageSet -> do - let currentPackages = (un PackageSet packageSet).packages - - -- Find packages uploaded in the last 24 hours that aren't already in the set - recentUploads <- findRecentUploads (Hours 24.0) - let - -- Filter out packages already in the set at the same or newer version - newOrUpdated = recentUploads # Map.filterWithKey \name version -> - case Map.lookup name currentPackages of - -- new package goes in - Nothing -> true - -- as do existing packages with a newer version - Just currentVersion -> version > currentVersion - - if Map.isEmpty newOrUpdated then - Log.info "No new packages for package set update." - else do - Log.info $ "Found " <> show (Map.size newOrUpdated) <> " candidates to validate" - - -- Pre-validate candidates to filter out packages with missing dependencies - manifestIndex <- Registry.readAllManifests - let candidates = PackageSets.validatePackageSetCandidates manifestIndex packageSet (map Just newOrUpdated) - - unless (Map.isEmpty candidates.rejected) do - Log.info $ "Some packages are not eligible for the package set:\n" <> PackageSets.printRejections candidates.rejected - - -- Only enqueue accepted packages (filter out removals, keep only updates) - let accepted = Map.catMaybes candidates.accepted - - if Map.isEmpty accepted then - Log.info "No packages passed validation for package set update." - else do - Log.info $ "Validated " <> show (Map.size accepted) <> " packages for package set update" - - -- Create a package set update payload with only validated packages - let - payload = Operation.PackageSetUpdate - { compiler: Nothing -- Use current compiler - , packages: map Just accepted -- Just version = add/update - } - rawPayload = stringifyJson Operation.packageSetOperationCodec payload - - -- Check if a similar job already exists - existingJob <- Db.selectPackageSetJobByPayload payload - case existingJob of - Just _ -> Log.debug "Package set job with same payload already exists" - Nothing -> do - -- No signature needed for package additions (only for compiler upgrades) - jobId <- Db.insertPackageSetJob { payload, rawPayload, signature: Nothing } - Log.info $ "Enqueued package set job " <> unwrap jobId - --- | Find the latest version of each package uploaded within the time limit -findRecentUploads :: Hours -> Run ServerEffects (Map PackageName Version) -findRecentUploads limit = do - allMetadata <- Registry.readAllMetadata - now <- nowUTC - - let - getLatestRecentVersion :: Metadata -> Maybe Version - getLatestRecentVersion (Metadata metadata) = do - let - recentVersions = Array.catMaybes $ flip map (Map.toUnfoldable metadata.published) - \(Tuple version { publishedTime }) -> if (DateTime.diff now publishedTime) <= limit then Just version else Nothing - Array.last $ Array.sort recentVersions - - pure $ Map.fromFoldable $ Array.catMaybes $ flip map (Map.toUnfoldable allMetadata) \(Tuple name metadata) -> - map (Tuple name) $ getLatestRecentVersion metadata - --- | Check for new tags on existing packages and enqueue publish jobs for --- | versions not yet published. This allows the registry to automatically --- | publish new versions of packages that are already in the registry. -scheduleDailyPublish :: Run ServerEffects Unit -scheduleDailyPublish = do - Log.info "Scheduler: checking for new package versions..." - - allMetadata <- Registry.readAllMetadata - let packages = Map.toUnfoldable allMetadata :: Array (Tuple PackageName Metadata) - - for_ packages \(Tuple name (Metadata metadata)) -> do - case metadata.location of - Git _ -> pure unit -- Skip non-GitHub packages for now - GitHub { owner, repo } -> do - GitHub.listTags { owner, repo } >>= case _ of - Left err -> do - Log.debug $ "Failed to fetch tags for " <> PackageName.print name <> ": " <> Octokit.printGitHubError err - Right tags -> do - let - -- Combine published and unpublished versions into a set - publishedVersions = Set.fromFoldable - $ Map.keys metadata.published - <> Map.keys metadata.unpublished - - -- Parse tags as versions and filter out already published ones - newVersions = Array.catMaybes $ tags <#> \tag -> - case LenientVersion.parse tag.name of - Left _ -> Nothing -- Not a valid version tag - Right result -> - let - version = LenientVersion.version result - in - if Set.member version publishedVersions then Nothing -- Already published - else Just { version, ref: tag.name } - - for_ newVersions \{ version, ref } -> - enqueuePublishJob allMetadata name (Metadata metadata) version ref - --- | Enqueue a publish job for a new package version discovered by the scheduler. --- | Attempts to find a compatible compiler by looking at the previous version's --- | dependencies. Falls back to the lowest compiler from the previous version if --- | no dependencies exist, or to the latest compiler if no previous version exists. -enqueuePublishJob :: Map PackageName Metadata -> PackageName -> Metadata -> Version -> String -> Run ServerEffects Unit -enqueuePublishJob allMetadata name (Metadata metadata) version ref = do - -- Check if a publish job already exists for this package version - existingJob <- Db.selectPublishJob name version - case existingJob of - Just _ -> Log.debug $ "Publish job already exists for " <> formatPackageVersion name version - Nothing -> do - -- Try to find a compatible compiler by looking at the previous version's dependencies - compiler <- case Map.findMax metadata.published of - Just { key: prevVersion, value: publishedInfo } -> do - -- Look up the manifest for the previous version to get its dependencies - maybeManifest <- Registry.readManifest name prevVersion - case maybeManifest of - Just (Manifest manifest) | not (Map.isEmpty manifest.dependencies) -> do - -- Use previous version's dependencies to find compatible compilers - -- Find the highest published version of each dependency within its range - let - depVersions :: Map PackageName Version - depVersions = Map.mapMaybeWithKey - ( \depName range -> - case Map.lookup depName allMetadata of - Just (Metadata depMeta) -> - Array.last $ Array.filter (Range.includes range) $ Array.sort $ Array.fromFoldable $ Map.keys depMeta.published - Nothing -> Nothing - ) - manifest.dependencies - - case compatibleCompilers allMetadata depVersions of - Just compilerSet -> pure $ NonEmptySet.min compilerSet - -- No intersection found, fall back to lowest compiler from previous version - Nothing -> pure $ NonEmptyArray.head publishedInfo.compilers - -- No manifest or no dependencies, fall back to lowest compiler from previous version - _ -> pure $ NonEmptyArray.head publishedInfo.compilers - Nothing -> - NonEmptyArray.last <$> PursVersions.pursVersions - let - payload = - { name - -- Don't specify location - use current metadata location at publish time. - -- This avoids race conditions with transfer jobs that may update the location. - , location: Nothing - , ref - , version - , compiler - , resolutions: Nothing - } - jobId <- Db.insertPublishJob { payload } - Log.info $ "Enqueued publish job " <> unwrap jobId <> " for " <> formatPackageVersion name version - --- | Given a set of package versions, determine the set of compilers that can be --- | used for all packages by intersecting their supported compiler ranges. -compatibleCompilers :: Map PackageName Metadata -> Map PackageName Version -> Maybe (NonEmptySet Version) -compatibleCompilers allMetadata resolutions = do - let - associated :: Array { compilers :: NonEmptyArray Version } - associated = Map.toUnfoldableUnordered resolutions # Array.mapMaybe \(Tuple depName depVersion) -> do - Metadata depMeta <- Map.lookup depName allMetadata - published <- Map.lookup depVersion depMeta.published - Just { compilers: published.compilers } - - case Array.uncons associated of - Nothing -> Nothing - Just { head, tail: [] } -> Just $ NonEmptySet.fromFoldable1 head.compilers - Just { head, tail } -> do - let foldFn prev = Set.intersection prev <<< Set.fromFoldable <<< _.compilers - NonEmptySet.fromFoldable $ Array.foldl foldFn (Set.fromFoldable head.compilers) tail diff --git a/flake.nix b/flake.nix index bbec4115..68dfe307 100644 --- a/flake.nix +++ b/flake.nix @@ -244,6 +244,8 @@ (writeShellScriptBin "spago-test-e2e" '' set -euo pipefail ${testEnv.envToExports testEnv.testEnv} + export PATH="${testEnv.testConfig.gitMock}/bin:$PATH" + export GIT_BINARY="${pkgs.git}/bin/git" exec spago run -p registry-app-e2e '') ]; diff --git a/nix/overlay.nix b/nix/overlay.nix index 8858c140..95876187 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -54,6 +54,10 @@ let module = "Registry.Scripts.ArchiveSeeder"; description = "Seed the registry archive with tarballs for deleted GitHub repos"; }; + daily-importer = { + module = "Registry.Scripts.DailyImporter"; + description = "Check for new package versions and submit publish jobs"; + }; legacy-importer = { module = "Registry.Scripts.LegacyImporter"; description = "Import packages from legacy registries (bower, psc-package, etc.)"; @@ -62,6 +66,14 @@ let module = "Registry.Scripts.PackageDeleter"; description = "Delete packages from the registry"; }; + package-set-updater = { + module = "Registry.Scripts.PackageSetUpdater"; + description = "Check for recent uploads and submit package set update jobs"; + }; + package-transferrer = { + module = "Registry.Scripts.PackageTransferrer"; + description = "Check for moved packages and submit transfer jobs"; + }; solver = { module = "Registry.Scripts.Solver"; description = "Run dependency solver against registry manifests"; diff --git a/scripts/src/ArchiveSeeder.purs b/scripts/src/ArchiveSeeder.purs index ca4fdcdd..827ddfdb 100644 --- a/scripts/src/ArchiveSeeder.purs +++ b/scripts/src/ArchiveSeeder.purs @@ -95,7 +95,6 @@ main = launchAff_ do runAppEffects <- do debouncer <- Registry.newDebouncer - repoLocks <- Registry.newRepoLocks let registryEnv = { pull: Git.Autostash @@ -104,8 +103,6 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef - , repoLocks - , process: Registry.ScriptArchiveSeeder } token <- Env.lookupRequired Env.githubToken diff --git a/scripts/src/CompilerVersions.purs b/scripts/src/CompilerVersions.purs index 176a200a..127d0b97 100644 --- a/scripts/src/CompilerVersions.purs +++ b/scripts/src/CompilerVersions.purs @@ -119,7 +119,6 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer - repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -129,8 +128,6 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef - , repoLocks - , process: Registry.ScriptCompilerVersions } -- Logging diff --git a/scripts/src/DailyImporter.purs b/scripts/src/DailyImporter.purs new file mode 100644 index 00000000..5130e956 --- /dev/null +++ b/scripts/src/DailyImporter.purs @@ -0,0 +1,268 @@ +-- | This script checks for new package versions by fetching GitHub tags for all +-- | packages in the registry. When a new version is discovered (a tag that hasn't +-- | been published or unpublished), it submits a publish job to the registry API. +-- | +-- | Run via Nix: +-- | nix run .#daily-importer -- --dry-run # Log what would be submitted +-- | nix run .#daily-importer -- --submit # Actually submit to the API +-- | +-- | Required environment variables: +-- | GITHUB_TOKEN - GitHub API token for fetching tags +-- | REGISTRY_API_URL - Registry API URL (default: https://registry.purescript.org) +module Registry.Scripts.DailyImporter where + +import Registry.App.Prelude + +import ArgParse.Basic (ArgParser) +import ArgParse.Basic as Arg +import Codec.JSON.DecodeError as CJ.DecodeError +import Data.Array as Array +import Data.Array.NonEmpty as NonEmptyArray +import Data.Codec.JSON as CJ +import Data.Map as Map +import Data.Set as Set +import Data.Set.NonEmpty (NonEmptySet) +import Data.Set.NonEmpty as NonEmptySet +import Effect.Aff as Aff +import Effect.Class.Console as Console +import Fetch (Method(..)) +import Fetch as Fetch +import JSON as JSON +import Node.Path as Path +import Node.Process as Process +import Registry.API.V1 as V1 +import Registry.App.CLI.Git as Git +import Registry.App.CLI.PursVersions as PursVersions +import Registry.App.Effect.Cache as Cache +import Registry.App.Effect.Env (RESOURCE_ENV) +import Registry.App.Effect.Env as Env +import Registry.App.Effect.GitHub (GITHUB) +import Registry.App.Effect.GitHub as GitHub +import Registry.App.Effect.Log (LOG) +import Registry.App.Effect.Log as Log +import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry as Registry +import Registry.App.Legacy.LenientVersion as LenientVersion +import Registry.Foreign.Octokit as Octokit +import Registry.Location (Location(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Registry.Range as Range +import Registry.Version as Version +import Run (AFF, EFFECT, Run) +import Run as Run +import Run.Except (EXCEPT) +import Run.Except as Except + +data Mode = DryRun | Submit + +derive instance Eq Mode + +parser :: ArgParser Mode +parser = Arg.choose "command" + [ Arg.flag [ "dry-run" ] + "Log what would be submitted without actually calling the API." + $> DryRun + , Arg.flag [ "submit" ] + "Submit publish jobs to the registry API." + $> Submit + ] + +main :: Effect Unit +main = launchAff_ do + args <- Array.drop 2 <$> liftEffect Process.argv + + let description = "Check for new package versions and submit publish jobs to the registry API." + mode <- case Arg.parseArgs "daily-importer" description parser args of + Left err -> Console.log (Arg.printArgError err) *> liftEffect (Process.exit' 1) + Right command -> pure command + + Env.loadEnvFile ".env" + resourceEnv <- Env.lookupResourceEnv + token <- Env.lookupRequired Env.githubToken + + githubCacheRef <- Cache.newCacheRef + registryCacheRef <- Cache.newCacheRef + let cache = Path.concat [ scratchDir, ".cache" ] + + octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl + debouncer <- Registry.newDebouncer + + let + registryEnv :: Registry.RegistryEnv + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + } + + runDailyImport mode resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Normal) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' + >>= case _ of + Left err -> do + Console.error $ "Error: " <> err + liftEffect $ Process.exit' 1 + Right _ -> pure unit + +type DailyImportEffects = (REGISTRY + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) + +runDailyImport :: Mode -> URL -> Run DailyImportEffects Unit +runDailyImport mode registryApiUrl = do + Log.info "Daily Importer: checking for new package versions..." + + allMetadata <- Registry.readAllMetadata + let packages = Map.toUnfoldable allMetadata :: Array (Tuple PackageName Metadata) + + Log.info $ "Checking " <> show (Array.length packages) <> " packages for new versions..." + + submitted <- for packages \(Tuple name (Metadata metadata)) -> do + case metadata.location of + Git _ -> pure 0 -- Skip non-GitHub packages for now + GitHub { owner, repo } -> do + GitHub.listTags { owner, repo } >>= case _ of + Left err -> do + Log.debug $ "Failed to fetch tags for " <> PackageName.print name <> ": " <> Octokit.printGitHubError err + pure 0 + Right tags -> do + let + -- Combine published and unpublished versions into a set + publishedVersions = Set.fromFoldable + $ Map.keys metadata.published + <> Map.keys metadata.unpublished + + -- Parse tags as versions and filter out already published ones + newVersions = Array.catMaybes $ tags <#> \tag -> + case LenientVersion.parse tag.name of + Left _ -> Nothing -- Not a valid version tag + Right result -> + let + version = LenientVersion.version result + in + if Set.member version publishedVersions then Nothing + else Just { version, ref: tag.name } + + -- Submit publish jobs for new versions + count <- for newVersions \{ version, ref } -> do + submitPublishJob mode registryApiUrl allMetadata name (Metadata metadata) version ref + + pure $ Array.length $ Array.filter identity count + + let totalSubmitted = Array.foldl (+) 0 submitted + Log.info $ "Daily Importer complete. Submitted " <> show totalSubmitted <> " publish jobs." + +-- | Submit a publish job for a new package version. +-- | Attempts to find a compatible compiler by looking at the previous version's +-- | dependencies. Falls back to the lowest compiler from the previous version if +-- | no dependencies exist, or to the latest compiler if no previous version exists. +submitPublishJob + :: Mode + -> URL + -> Map PackageName Metadata + -> PackageName + -> Metadata + -> Version + -> String + -> Run DailyImportEffects Boolean +submitPublishJob mode registryApiUrl allMetadata name (Metadata metadata) version ref = do + let formatted = formatPackageVersion name version + + -- Determine the appropriate compiler version + compiler <- case Map.findMax metadata.published of + Just { key: prevVersion, value: publishedInfo } -> do + -- Look up the manifest for the previous version to get its dependencies + maybeManifest <- Registry.readManifest name prevVersion + case maybeManifest of + Just (Manifest manifest) | not (Map.isEmpty manifest.dependencies) -> do + -- Use previous version's dependencies to find compatible compilers + let + depVersions :: Map PackageName Version + depVersions = Map.mapMaybeWithKey + ( \depName range -> + case Map.lookup depName allMetadata of + Just (Metadata depMeta) -> + Array.last $ Array.filter (Range.includes range) $ Array.sort $ Array.fromFoldable $ Map.keys depMeta.published + Nothing -> Nothing + ) + manifest.dependencies + + case compatibleCompilers allMetadata depVersions of + Just compilerSet -> pure $ NonEmptySet.min compilerSet + -- No intersection found, fall back to lowest compiler from previous version + Nothing -> pure $ NonEmptyArray.head publishedInfo.compilers + -- No manifest or no dependencies, fall back to lowest compiler from previous version + _ -> pure $ NonEmptyArray.head publishedInfo.compilers + Nothing -> + NonEmptyArray.last <$> PursVersions.pursVersions + + let + payload :: Operation.PublishData + payload = + { name + , version + , location: Nothing -- Use current metadata location at publish time + , ref + , compiler + , resolutions: Nothing + } + + case mode of + DryRun -> do + Log.info $ "[DRY RUN] Would submit publish job for " <> formatted <> " with compiler " <> Version.print compiler + pure true + + Submit -> do + Log.info $ "Submitting publish job for " <> formatted <> " with compiler " <> Version.print compiler + result <- Run.liftAff $ submitJob (registryApiUrl <> "/v1/publish") payload + case result of + Left err -> do + Log.error $ "Failed to submit publish job for " <> formatted <> ": " <> err + pure false + Right { jobId } -> do + Log.info $ "Submitted publish job " <> unwrap jobId <> " for " <> formatted + pure true + +-- | Submit a job to the registry API +submitJob :: String -> Operation.PublishData -> Aff (Either String V1.JobCreatedResponse) +submitJob url payload = do + let body = JSON.print $ CJ.encode Operation.publishCodec payload + result <- Aff.attempt $ Fetch.fetch url + { method: POST + , headers: { "Content-Type": "application/json" } + , body + } + case result of + Left err -> pure $ Left $ "Network error: " <> Aff.message err + Right response -> do + responseBody <- response.text + if response.status >= 200 && response.status < 300 then + case JSON.parse responseBody >>= \json -> lmap CJ.DecodeError.print (CJ.decode V1.jobCreatedResponseCodec json) of + Left err -> pure $ Left $ "Failed to parse response: " <> err + Right r -> pure $ Right r + else + pure $ Left $ "HTTP " <> show response.status <> ": " <> responseBody + +-- | Given a set of package versions, determine the set of compilers that can be +-- | used for all packages by intersecting their supported compiler ranges. +compatibleCompilers :: Map PackageName Metadata -> Map PackageName Version -> Maybe (NonEmptySet Version) +compatibleCompilers allMetadata resolutions = do + let + associated :: Array { compilers :: NonEmptyArray Version } + associated = Map.toUnfoldableUnordered resolutions # Array.mapMaybe \(Tuple depName depVersion) -> do + Metadata depMeta <- Map.lookup depName allMetadata + published <- Map.lookup depVersion depMeta.published + Just { compilers: published.compilers } + + case Array.uncons associated of + Nothing -> Nothing + Just { head, tail: [] } -> Just $ NonEmptySet.fromFoldable1 head.compilers + Just { head, tail } -> do + let foldFn prev = Set.intersection prev <<< Set.fromFoldable <<< _.compilers + NonEmptySet.fromFoldable $ Array.foldl foldFn (Set.fromFoldable head.compilers) tail diff --git a/scripts/src/LegacyImporter.purs b/scripts/src/LegacyImporter.purs index 2decb34b..7e65c48c 100644 --- a/scripts/src/LegacyImporter.purs +++ b/scripts/src/LegacyImporter.purs @@ -189,7 +189,6 @@ main = launchAff_ do -- uploaded and manifests and metadata are written, committed, and pushed. runAppEffects <- do debouncer <- Registry.newDebouncer - repoLocks <- Registry.newRepoLocks let registryEnv pull write = { pull @@ -198,8 +197,6 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef - , repoLocks - , process: Registry.ScriptLegacyImporter } case mode of DryRun -> do diff --git a/scripts/src/PackageDeleter.purs b/scripts/src/PackageDeleter.purs index d93d268b..257a7b1a 100644 --- a/scripts/src/PackageDeleter.purs +++ b/scripts/src/PackageDeleter.purs @@ -122,7 +122,6 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer - repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -132,8 +131,6 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef - , repoLocks - , process: Registry.ScriptPackageDeleter } -- Logging diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs new file mode 100644 index 00000000..3b3f9aee --- /dev/null +++ b/scripts/src/PackageSetUpdater.purs @@ -0,0 +1,240 @@ +-- | This script checks for packages recently uploaded to the registry and +-- | submits package set update jobs to add them to the package set. +-- | +-- | Run via Nix: +-- | nix run .#package-set-updater -- --dry-run # Log what would be submitted +-- | nix run .#package-set-updater -- --submit # Actually submit to the API +-- | +-- | Required environment variables: +-- | GITHUB_TOKEN - GitHub API token +-- | PACCHETTIBOTTI_ED25519 - Private key for signing (only for --submit) +-- | REGISTRY_API_URL - Registry API URL (default: https://registry.purescript.org) +module Registry.Scripts.PackageSetUpdater where + +import Registry.App.Prelude + +import ArgParse.Basic (ArgParser) +import ArgParse.Basic as Arg +import Codec.JSON.DecodeError as CJ.DecodeError +import Data.Array as Array +import Data.Codec.JSON as CJ +import Data.DateTime as DateTime +import Data.Map as Map +import Data.Time.Duration (Hours(..)) +import Effect.Aff as Aff +import Effect.Class.Console as Console +import Fetch (Method(..)) +import Fetch as Fetch +import JSON as JSON +import Node.Path as Path +import Node.Process as Process +import Registry.API.V1 as V1 +import Registry.App.Auth as Auth +import Registry.App.CLI.Git as Git +import Registry.App.Effect.Cache as Cache +import Registry.App.Effect.Env (RESOURCE_ENV) +import Registry.App.Effect.Env as Env +import Registry.App.Effect.GitHub (GITHUB) +import Registry.App.Effect.GitHub as GitHub +import Registry.App.Effect.Log (LOG) +import Registry.App.Effect.Log as Log +import Registry.App.Effect.PackageSets as PackageSets +import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry as Registry +import Registry.Foreign.Octokit as Octokit +import Registry.Operation (PackageSetOperation(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Registry.PackageSet (PackageSet(..)) +import Registry.Version as Version +import Run (AFF, EFFECT, Run) +import Run as Run +import Run.Except (EXCEPT) +import Run.Except as Except + +data Mode = DryRun | Submit + +derive instance Eq Mode + +parser :: ArgParser Mode +parser = Arg.choose "command" + [ Arg.flag [ "dry-run" ] + "Log what would be submitted without actually calling the API." + $> DryRun + , Arg.flag [ "submit" ] + "Submit package set update jobs to the registry API." + $> Submit + ] + +main :: Effect Unit +main = launchAff_ do + args <- Array.drop 2 <$> liftEffect Process.argv + + let description = "Check for recent uploads and submit package set update jobs to the registry API." + mode <- case Arg.parseArgs "package-set-updater" description parser args of + Left err -> Console.log (Arg.printArgError err) *> liftEffect (Process.exit' 1) + Right command -> pure command + + Env.loadEnvFile ".env" + resourceEnv <- Env.lookupResourceEnv + token <- Env.lookupRequired Env.githubToken + + -- Only require pacchettibotti keys in submit mode + maybePrivateKey <- case mode of + DryRun -> pure Nothing + Submit -> Just <$> Env.lookupRequired Env.pacchettibottiED25519 + + githubCacheRef <- Cache.newCacheRef + registryCacheRef <- Cache.newCacheRef + let cache = Path.concat [ scratchDir, ".cache" ] + + octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl + debouncer <- Registry.newDebouncer + + let + registryEnv :: Registry.RegistryEnv + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + } + + runPackageSetUpdater mode maybePrivateKey resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Normal) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' + >>= case _ of + Left err -> do + Console.error $ "Error: " <> err + liftEffect $ Process.exit' 1 + Right _ -> pure unit + +type PackageSetUpdaterEffects = (REGISTRY + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) + +runPackageSetUpdater :: Mode -> Maybe String -> URL -> Run PackageSetUpdaterEffects Unit +runPackageSetUpdater mode maybePrivateKey registryApiUrl = do + Log.info "Package Set Updater: checking for recent uploads..." + + -- Get the current package set + latestPackageSet <- Registry.readLatestPackageSet >>= case _ of + Nothing -> do + Log.warn "No package set found, skipping package set updates" + pure Nothing + Just set -> pure (Just set) + + for_ latestPackageSet \packageSet -> do + let currentPackages = (un PackageSet packageSet).packages + + -- Find packages uploaded in the last 24 hours + recentUploads <- findRecentUploads (Hours 24.0) + let + -- Filter out packages already in the set at the same or newer version + newOrUpdated = recentUploads # Map.filterWithKey \name version -> + case Map.lookup name currentPackages of + Nothing -> true -- new package + Just currentVersion -> version > currentVersion -- upgrade + + if Map.isEmpty newOrUpdated then + Log.info "No new packages for package set update." + else do + Log.info $ "Found " <> show (Map.size newOrUpdated) <> " candidates to validate" + + -- Pre-validate candidates to filter out packages with missing dependencies + manifestIndex <- Registry.readAllManifests + let candidates = PackageSets.validatePackageSetCandidates manifestIndex packageSet (map Just newOrUpdated) + + unless (Map.isEmpty candidates.rejected) do + Log.info $ "Some packages are not eligible for the package set:\n" <> PackageSets.printRejections candidates.rejected + + -- Only include accepted packages (filter out removals, keep only updates) + let accepted = Map.catMaybes candidates.accepted + + if Map.isEmpty accepted then + Log.info "No packages passed validation for package set update." + else do + Log.info $ "Validated " <> show (Map.size accepted) <> " packages for package set update" + + -- Create a package set update payload + let + payload :: Operation.PackageSetUpdateData + payload = + { compiler: Nothing -- Use current compiler + , packages: map Just accepted -- Just version = add/update + } + + case mode of + DryRun -> do + Log.info $ "[DRY RUN] Would submit package set update with packages:" + for_ (Map.toUnfoldable accepted :: Array _) \(Tuple name version) -> + Log.info $ " - " <> PackageName.print name <> "@" <> Version.print version + + Submit -> do + privateKey <- case maybePrivateKey of + Nothing -> Except.throw "PACCHETTIBOTTI_ED25519 required for --submit mode" + Just pk -> pure pk + + -- Sign the payload with pacchettibotti keys + let rawPayload = JSON.print $ CJ.encode Operation.packageSetUpdateCodec payload + signature <- case Auth.signPayload { privateKey, rawPayload } of + Left err -> Except.throw $ "Error signing package set update: " <> err + Right sig -> pure sig + + let + request :: Operation.PackageSetUpdateRequest + request = + { payload: PackageSetUpdate payload + , rawPayload + , signature: Just signature + } + + Log.info $ "Submitting package set update..." + result <- Run.liftAff $ submitPackageSetJob (registryApiUrl <> "/v1/package-sets") request + case result of + Left err -> do + Log.error $ "Failed to submit package set job: " <> err + Right { jobId } -> do + Log.info $ "Submitted package set job " <> unwrap jobId + +-- | Find the latest version of each package uploaded within the time limit +findRecentUploads :: Hours -> Run PackageSetUpdaterEffects (Map PackageName Version) +findRecentUploads limit = do + allMetadata <- Registry.readAllMetadata + now <- nowUTC + + let + getLatestRecentVersion :: Metadata -> Maybe Version + getLatestRecentVersion (Metadata metadata) = do + let + recentVersions = Array.catMaybes $ flip map (Map.toUnfoldable metadata.published) + \(Tuple version { publishedTime }) -> + if (DateTime.diff now publishedTime) <= limit then Just version else Nothing + Array.last $ Array.sort recentVersions + + pure $ Map.fromFoldable $ Array.catMaybes $ flip map (Map.toUnfoldable allMetadata) \(Tuple name metadata) -> + map (Tuple name) $ getLatestRecentVersion metadata + +-- | Submit a package set job to the registry API +submitPackageSetJob :: String -> Operation.PackageSetUpdateRequest -> Aff (Either String V1.JobCreatedResponse) +submitPackageSetJob url request = do + let body = JSON.print $ CJ.encode Operation.packageSetUpdateRequestCodec request + result <- Aff.attempt $ Fetch.fetch url + { method: POST + , headers: { "Content-Type": "application/json" } + , body + } + case result of + Left err -> pure $ Left $ "Network error: " <> Aff.message err + Right response -> do + responseBody <- response.text + if response.status >= 200 && response.status < 300 then + case JSON.parse responseBody >>= \json -> lmap CJ.DecodeError.print (CJ.decode V1.jobCreatedResponseCodec json) of + Left err -> pure $ Left $ "Failed to parse response: " <> err + Right r -> pure $ Right r + else + pure $ Left $ "HTTP " <> show response.status <> ": " <> responseBody diff --git a/scripts/src/PackageTransferrer.purs b/scripts/src/PackageTransferrer.purs new file mode 100644 index 00000000..8826dfc2 --- /dev/null +++ b/scripts/src/PackageTransferrer.purs @@ -0,0 +1,235 @@ +-- | This script checks for packages that have moved to a new GitHub location +-- | and submits transfer jobs to update their registered location. +-- | +-- | Run via Nix: +-- | nix run .#package-transferrer -- --dry-run # Log what would be submitted +-- | nix run .#package-transferrer -- --submit # Actually submit to the API +-- | +-- | Required environment variables: +-- | GITHUB_TOKEN - GitHub API token for fetching tags +-- | PACCHETTIBOTTI_ED25519 - Private key for signing (only for --submit) +-- | REGISTRY_API_URL - Registry API URL (default: https://registry.purescript.org) +module Registry.Scripts.PackageTransferrer where + +import Registry.App.Prelude + +import ArgParse.Basic (ArgParser) +import ArgParse.Basic as Arg +import Codec.JSON.DecodeError as CJ.DecodeError +import Data.Array as Array +import Data.Codec.JSON as CJ +import Data.Map as Map +import Data.String as String +import Effect.Aff as Aff +import Effect.Class.Console as Console +import Fetch (Method(..)) +import Fetch as Fetch +import JSON as JSON +import Node.Path as Path +import Node.Process as Process +import Registry.API.V1 as V1 +import Registry.App.Auth as Auth +import Registry.App.CLI.Git as Git +import Registry.App.Effect.Cache as Cache +import Registry.App.Effect.Env (RESOURCE_ENV) +import Registry.App.Effect.Env as Env +import Registry.App.Effect.GitHub (GITHUB) +import Registry.App.Effect.GitHub as GitHub +import Registry.App.Effect.Log (LOG) +import Registry.App.Effect.Log as Log +import Registry.App.Effect.Registry (REGISTRY) +import Registry.App.Effect.Registry as Registry +import Registry.Foreign.Octokit as Octokit +import Registry.Location (Location(..)) +import Registry.Operation (AuthenticatedPackageOperation(..)) +import Registry.Operation as Operation +import Registry.PackageName as PackageName +import Run (AFF, EFFECT, Run) +import Run as Run +import Run.Except (EXCEPT) +import Run.Except as Except + +data Mode = DryRun | Submit + +derive instance Eq Mode + +parser :: ArgParser Mode +parser = Arg.choose "command" + [ Arg.flag [ "dry-run" ] + "Log what would be submitted without actually calling the API." + $> DryRun + , Arg.flag [ "submit" ] + "Submit transfer jobs to the registry API." + $> Submit + ] + +main :: Effect Unit +main = launchAff_ do + args <- Array.drop 2 <$> liftEffect Process.argv + + let description = "Check for moved packages and submit transfer jobs to the registry API." + mode <- case Arg.parseArgs "package-transferrer" description parser args of + Left err -> Console.log (Arg.printArgError err) *> liftEffect (Process.exit' 1) + Right command -> pure command + + Env.loadEnvFile ".env" + resourceEnv <- Env.lookupResourceEnv + token <- Env.lookupRequired Env.githubToken + + -- Only require pacchettibotti keys in submit mode + maybePrivateKey <- case mode of + DryRun -> pure Nothing + Submit -> Just <$> Env.lookupRequired Env.pacchettibottiED25519 + + githubCacheRef <- Cache.newCacheRef + registryCacheRef <- Cache.newCacheRef + let cache = Path.concat [ scratchDir, ".cache" ] + + octokit <- Octokit.newOctokit token resourceEnv.githubApiUrl + debouncer <- Registry.newDebouncer + + let + registryEnv :: Registry.RegistryEnv + registryEnv = + { pull: Git.Autostash + , write: Registry.ReadOnly + , repos: Registry.defaultRepos + , workdir: scratchDir + , debouncer + , cacheRef: registryCacheRef + } + + runPackageTransferrer mode maybePrivateKey resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Normal) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' + >>= case _ of + Left err -> do + Console.error $ "Error: " <> err + liftEffect $ Process.exit' 1 + Right _ -> pure unit + +type PackageTransferrerEffects = (REGISTRY + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) + +runPackageTransferrer :: Mode -> Maybe String -> URL -> Run PackageTransferrerEffects Unit +runPackageTransferrer mode maybePrivateKey registryApiUrl = do + Log.info "Package Transferrer: checking for package transfers..." + allMetadata <- Registry.readAllMetadata + + -- Check each package for location changes + transfersNeeded <- Array.catMaybes <$> for (Map.toUnfoldable allMetadata) \(Tuple name (Metadata metadata)) -> + case metadata.location of + Git _ -> pure Nothing -- Skip non-GitHub packages + GitHub registered -> do + -- Fetch tags to see if repo has moved + GitHub.listTags { owner: registered.owner, repo: registered.repo } >>= case _ of + Left _ -> pure Nothing -- Can't fetch tags, skip + Right tags | Array.null tags -> pure Nothing -- No tags, skip + Right tags -> case Array.head tags of + Nothing -> pure Nothing + Just tag -> + -- Parse the tag URL to get actual current location + case tagUrlToRepoUrl tag.url of + Nothing -> pure Nothing + Just actual + | locationsMatch registered actual -> pure Nothing -- No change + | otherwise -> pure $ Just + { name + , newLocation: GitHub { owner: actual.owner, repo: actual.repo, subdir: registered.subdir } + } + + case Array.length transfersNeeded of + 0 -> Log.info "No packages require transferring." + n -> do + Log.info $ show n <> " packages need transferring" + for_ transfersNeeded \{ name, newLocation } -> + submitTransferJob mode maybePrivateKey registryApiUrl name newLocation + +-- | Parse GitHub API tag URL to extract owner/repo +-- | Example: https://api.github.com/repos/octocat/Hello-World/commits/abc123 +tagUrlToRepoUrl :: String -> Maybe { owner :: String, repo :: String } +tagUrlToRepoUrl url = do + noPrefix <- String.stripPrefix (String.Pattern "https://api.github.com/repos/") url + case Array.take 2 $ String.split (String.Pattern "/") noPrefix of + [ owner, repo ] -> Just { owner, repo: String.toLower repo } + _ -> Nothing + +-- | Case-insensitive comparison of GitHub locations +locationsMatch :: forall r. { owner :: String, repo :: String | r } -> { owner :: String, repo :: String } -> Boolean +locationsMatch loc1 loc2 = + String.toLower loc1.owner == String.toLower loc2.owner + && String.toLower loc1.repo + == String.toLower loc2.repo + +-- | Submit a transfer job for a package that has moved +submitTransferJob + :: Mode + -> Maybe String + -> URL + -> PackageName + -> Location + -> Run PackageTransferrerEffects Unit +submitTransferJob mode maybePrivateKey registryApiUrl name newLocation = do + let formatted = PackageName.print name + + case mode of + DryRun -> do + let + locStr = case newLocation of + GitHub { owner, repo } -> owner <> "/" <> repo + Git { url } -> url + Log.info $ "[DRY RUN] Would submit transfer job for " <> formatted <> " to " <> locStr + + Submit -> do + privateKey <- case maybePrivateKey of + Nothing -> Except.throw "PACCHETTIBOTTI_ED25519 required for --submit mode" + Just pk -> pure pk + + let + payload :: Operation.TransferData + payload = { name, newLocation } + rawPayload = JSON.print $ CJ.encode Operation.transferCodec payload + + -- Sign the payload with pacchettibotti keys + signature <- case Auth.signPayload { privateKey, rawPayload } of + Left err -> Except.throw $ "Error signing transfer for " <> formatted <> ": " <> err + Right sig -> pure sig + + let + authenticatedData :: Operation.AuthenticatedData + authenticatedData = + { payload: Transfer payload + , rawPayload + , signature + } + + Log.info $ "Submitting transfer job for " <> formatted + result <- Run.liftAff $ submitJob (registryApiUrl <> "/v1/transfer") authenticatedData + case result of + Left err -> do + Log.error $ "Failed to submit transfer job for " <> formatted <> ": " <> err + Right { jobId } -> do + Log.info $ "Submitted transfer job " <> unwrap jobId <> " for " <> formatted + +-- | Submit a transfer job to the registry API +submitJob :: String -> Operation.AuthenticatedData -> Aff (Either String V1.JobCreatedResponse) +submitJob url authData = do + let body = JSON.print $ CJ.encode Operation.authenticatedCodec authData + result <- Aff.attempt $ Fetch.fetch url + { method: POST + , headers: { "Content-Type": "application/json" } + , body + } + case result of + Left err -> pure $ Left $ "Network error: " <> Aff.message err + Right response -> do + responseBody <- response.text + if response.status >= 200 && response.status < 300 then + case JSON.parse responseBody >>= \json -> lmap CJ.DecodeError.print (CJ.decode V1.jobCreatedResponseCodec json) of + Left err -> pure $ Left $ "Failed to parse response: " <> err + Right r -> pure $ Right r + else + pure $ Left $ "HTTP " <> show response.status <> ": " <> responseBody diff --git a/scripts/src/Solver.purs b/scripts/src/Solver.purs index a61bcacf..cd047a1c 100644 --- a/scripts/src/Solver.purs +++ b/scripts/src/Solver.purs @@ -117,7 +117,6 @@ main = launchAff_ do FS.Extra.ensureDirectory cache debouncer <- Registry.newDebouncer - repoLocks <- Registry.newRepoLocks let registryEnv pull write = { pull @@ -126,8 +125,6 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef - , repoLocks - , process: Registry.ScriptSolver } resourceEnv <- Env.lookupResourceEnv token <- Env.lookupRequired Env.githubToken diff --git a/scripts/src/VerifyIntegrity.purs b/scripts/src/VerifyIntegrity.purs index ac79bd7e..97aef379 100644 --- a/scripts/src/VerifyIntegrity.purs +++ b/scripts/src/VerifyIntegrity.purs @@ -79,7 +79,6 @@ main = launchAff_ do -- Registry debouncer <- Registry.newDebouncer - repoLocks <- Registry.newRepoLocks let registryEnv :: Registry.RegistryEnv registryEnv = @@ -89,8 +88,6 @@ main = launchAff_ do , workdir: scratchDir , debouncer , cacheRef: registryCacheRef - , repoLocks - , process: Registry.ScriptVerifyIntegrity } -- Logging diff --git a/spago.lock b/spago.lock index ea939dde..840903c3 100644 --- a/spago.lock +++ b/spago.lock @@ -319,8 +319,10 @@ "registry-app", "registry-foreign", "registry-lib", + "registry-scripts", "registry-test-utils", "routing-duplex", + "run", "spec", "spec-node", "strings", @@ -332,6 +334,7 @@ "ansi", "argonaut-codecs", "argonaut-core", + "argparse-basic", "arraybuffer-types", "arrays", "assert", @@ -435,6 +438,7 @@ "registry-app", "registry-foreign", "registry-lib", + "registry-scripts", "registry-test-utils", "routing-duplex", "run", From a284c23122ad33241e205ca20cfd2b0e4b990ad4 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sat, 17 Jan 2026 15:49:36 +0200 Subject: [PATCH 10/20] Format --- app-e2e/src/Test/E2E/Scripts.purs | 50 +++++++++++++++---------------- app/src/App/Effect/Registry.purs | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/app-e2e/src/Test/E2E/Scripts.purs b/app-e2e/src/Test/E2E/Scripts.purs index e4b0ef9f..e4dd0751 100644 --- a/app-e2e/src/Test/E2E/Scripts.purs +++ b/app-e2e/src/Test/E2E/Scripts.purs @@ -34,7 +34,7 @@ import Registry.Version as Version import Run as Run import Run.Except as Except import Test.E2E.Support.Client as Client -import Test.E2E.Support.Env (E2ESpec, E2E) +import Test.E2E.Support.Env (E2E, E2ESpec) import Test.Spec as Spec spec :: E2ESpec @@ -167,14 +167,14 @@ runDailyImporterScript = do , cacheRef: registryCacheRef } - result <- liftAff $ - DailyImporter.runDailyImport DailyImporter.Submit resourceEnv.registryApiUrl - # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - # Log.interpret (Log.handleTerminal Quiet) - # Env.runResourceEnv resourceEnv - # Run.runBaseAff' + result <- liftAff + $ DailyImporter.runDailyImport DailyImporter.Submit resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Quiet) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' case result of Left err -> liftAff $ Aff.throwError $ Aff.error $ "DailyImporter failed: " <> err @@ -210,14 +210,14 @@ runPackageTransferrerScript = do , cacheRef: registryCacheRef } - result <- liftAff $ - PackageTransferrer.runPackageTransferrer PackageTransferrer.Submit (Just privateKey) resourceEnv.registryApiUrl - # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - # Log.interpret (Log.handleTerminal Quiet) - # Env.runResourceEnv resourceEnv - # Run.runBaseAff' + result <- liftAff + $ PackageTransferrer.runPackageTransferrer PackageTransferrer.Submit (Just privateKey) resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Quiet) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' case result of Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageTransferrer failed: " <> err @@ -253,14 +253,14 @@ runPackageSetUpdaterScript = do , cacheRef: registryCacheRef } - result <- liftAff $ - PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit (Just privateKey) resourceEnv.registryApiUrl - # Except.runExcept - # Registry.interpret (Registry.handle registryEnv) - # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) - # Log.interpret (Log.handleTerminal Quiet) - # Env.runResourceEnv resourceEnv - # Run.runBaseAff' + result <- liftAff + $ PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit (Just privateKey) resourceEnv.registryApiUrl + # Except.runExcept + # Registry.interpret (Registry.handle registryEnv) + # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) + # Log.interpret (Log.handleTerminal Quiet) + # Env.runResourceEnv resourceEnv + # Run.runBaseAff' case result of Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageSetUpdater failed: " <> err diff --git a/app/src/App/Effect/Registry.purs b/app/src/App/Effect/Registry.purs index e9b123fb..da696429 100644 --- a/app/src/App/Effect/Registry.purs +++ b/app/src/App/Effect/Registry.purs @@ -22,7 +22,6 @@ import Effect.Ref as Ref import JSON as JSON import Node.FS.Aff as FS.Aff import Node.Path as Path -import Registry.Foreign.FSExtra as FS.Extra import Registry.App.CLI.Git (GitResult) import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache (class MemoryEncodable, Cache, CacheRef, MemoryEncoding(..)) @@ -35,6 +34,7 @@ import Registry.App.Legacy.PackageSet (PscTag(..)) import Registry.App.Legacy.PackageSet as Legacy.PackageSet import Registry.App.Legacy.Types (legacyPackageSetCodec) import Registry.Constants as Constants +import Registry.Foreign.FSExtra as FS.Extra import Registry.Foreign.FastGlob as FastGlob import Registry.Foreign.Octokit (Address) import Registry.Foreign.Octokit as Octokit From c0e7cd74b660b499059e2397019ddda795d0673e Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sat, 17 Jan 2026 17:55:27 +0200 Subject: [PATCH 11/20] Address review feedback --- app-e2e/src/Test/E2E/Endpoint/Startup.purs | 20 ++-- app-e2e/src/Test/E2E/Scripts.purs | 120 +++++++-------------- 2 files changed, 49 insertions(+), 91 deletions(-) diff --git a/app-e2e/src/Test/E2E/Endpoint/Startup.purs b/app-e2e/src/Test/E2E/Endpoint/Startup.purs index 0b60c2fe..0bc03712 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Startup.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Startup.purs @@ -1,7 +1,5 @@ -- | E2E tests for server startup behavior (non-scheduler). -- | --- | - checkIfNewCompiler: Detects new compiler and enqueues matrix jobs --- | -- | IMPORTANT: These tests must run BEFORE resetTestState is called, since -- | the jobs are created at server startup and would be cleared. module Test.E2E.Endpoint.Startup (spec) where @@ -9,10 +7,11 @@ module Test.E2E.Endpoint.Startup (spec) where import Registry.App.Prelude import Data.Array as Array +import Data.String as String import Registry.API.V1 (Job(..)) import Registry.PackageName as PackageName import Registry.Test.Assert as Assert -import Registry.Version as Version +import Registry.Test.Utils as Utils import Test.E2E.Support.Client as Client import Test.E2E.Support.Env (E2ESpec) import Test.Spec as Spec @@ -30,24 +29,21 @@ spec = do isNewCompilerMatrixJob :: Job -> Boolean isNewCompilerMatrixJob = case _ of MatrixJob { compilerVersion } -> - compilerVersion == unsafeFromRight (Version.parse "0.15.11") + compilerVersion == Utils.unsafeVersion "0.15.11" _ -> false matrixJobs = Array.filter isNewCompilerMatrixJob jobs -- Get package names from matrix jobs - matrixPackages = Array.mapMaybe - ( \j -> case j of - MatrixJob { packageName } -> Just packageName - _ -> Nothing - ) - matrixJobs + matrixPackages = matrixJobs # Array.mapMaybe case _ of + MatrixJob { packageName } -> Just packageName + _ -> Nothing -- Should have matrix jobs for packages with no dependencies -- prelude has no dependencies, so it should get a matrix job - let preludeName = unsafeFromRight (PackageName.parse "prelude") + let preludeName = Utils.unsafePackageName "prelude" unless (Array.elem preludeName matrixPackages) do Assert.fail $ "Expected matrix job for prelude with compiler 0.15.11, found: " <> show (Array.length matrixJobs) <> " matrix jobs for packages: " - <> show (map PackageName.print matrixPackages) + <> String.joinWith ", " (map PackageName.print matrixPackages) diff --git a/app-e2e/src/Test/E2E/Scripts.purs b/app-e2e/src/Test/E2E/Scripts.purs index e4dd0751..68cb2960 100644 --- a/app-e2e/src/Test/E2E/Scripts.purs +++ b/app-e2e/src/Test/E2E/Scripts.purs @@ -30,6 +30,7 @@ import Registry.Scripts.DailyImporter as DailyImporter import Registry.Scripts.PackageSetUpdater as PackageSetUpdater import Registry.Scripts.PackageTransferrer as PackageTransferrer import Registry.Test.Assert as Assert +import Registry.Test.Utils as Utils import Registry.Version as Version import Run as Run import Run.Except as Except @@ -37,6 +38,19 @@ import Test.E2E.Support.Client as Client import Test.E2E.Support.Env (E2E, E2ESpec) import Test.Spec as Spec +-- | Constants for repeated package names and versions in tests +typeEqualityName :: PackageName.PackageName +typeEqualityName = Utils.unsafePackageName "type-equality" + +typeEqualityV401 :: Version.Version +typeEqualityV401 = Utils.unsafeVersion "4.0.1" + +typeEqualityV402 :: Version.Version +typeEqualityV402 = Utils.unsafeVersion "4.0.2" + +compiler01510 :: Version.Version +compiler01510 = Utils.unsafeVersion "0.15.10" + spec :: E2ESpec spec = do Spec.describe "DailyImporter" do @@ -49,9 +63,7 @@ spec = do isTypeEqualityPublishJob :: Job -> Boolean isTypeEqualityPublishJob = case _ of PublishJob { packageName, packageVersion } -> - packageName == unsafeFromRight (PackageName.parse "type-equality") - && packageVersion - == unsafeFromRight (Version.parse "4.0.2") + packageName == typeEqualityName && packageVersion == typeEqualityV402 _ -> false typeEqualityJob = Array.find isTypeEqualityPublishJob jobs @@ -59,8 +71,7 @@ spec = do case typeEqualityJob of Just (PublishJob { payload }) -> do -- Verify compiler selection logic - let expectedCompiler = unsafeFromRight (Version.parse "0.15.10") - when (payload.compiler /= expectedCompiler) do + when (payload.compiler /= compiler01510) do Assert.fail $ "Expected compiler 0.15.10 but got " <> Version.print payload.compiler Just _ -> Assert.fail "Expected PublishJob but got different job type" Nothing -> do @@ -79,9 +90,7 @@ spec = do isDuplicateJob :: Job -> Boolean isDuplicateJob = case _ of PublishJob { packageName, packageVersion } -> - packageName == unsafeFromRight (PackageName.parse "type-equality") - && packageVersion - == unsafeFromRight (Version.parse "4.0.1") + packageName == typeEqualityName && packageVersion == typeEqualityV401 _ -> false duplicateJob = Array.find isDuplicateJob jobs @@ -99,12 +108,12 @@ spec = do isTypeEqualityTransferJob :: Job -> Boolean isTypeEqualityTransferJob = case _ of TransferJob { packageName } -> - packageName == unsafeFromRight (PackageName.parse "type-equality") + packageName == typeEqualityName _ -> false case Array.find isTypeEqualityTransferJob jobs of Just (TransferJob { packageName, payload }) -> do -- Verify packageName - when (packageName /= unsafeFromRight (PackageName.parse "type-equality")) do + when (packageName /= typeEqualityName) do Assert.fail $ "Wrong package name: " <> PackageName.print packageName -- Verify newLocation in payload case payload.payload of @@ -131,31 +140,34 @@ spec = do Just (PackageSetJob { payload }) -> case payload of Operation.PackageSetUpdate { packages } -> - case Map.lookup (unsafeFromRight $ PackageName.parse "type-equality") packages of + case Map.lookup typeEqualityName packages of Just (Just _) -> pure unit _ -> Assert.fail "Expected type-equality in package set update" Just _ -> Assert.fail "Expected PackageSetJob but got different job type" Nothing -> Assert.fail "Expected package set job to be enqueued" --- | Run the DailyImporter script in Submit mode -runDailyImporterScript :: E2E Unit -runDailyImporterScript = do - { stateDir } <- ask - - -- Set up environment +-- | Common environment for running registry scripts in E2E tests +type ScriptSetup = + { privateKey :: String + , resourceEnv :: Env.ResourceEnv + , registryEnv :: Registry.RegistryEnv + , octokit :: Octokit.Octokit + , cache :: FilePath + , githubCacheRef :: Cache.CacheRef + } + +-- | Set up common environment for running registry scripts +setupScript :: E2E ScriptSetup +setupScript = do + { stateDir, privateKey } <- ask liftEffect $ Process.chdir stateDir - - -- Get resource env from environment variables resourceEnv <- liftEffect Env.lookupResourceEnv token <- liftEffect $ Env.lookupRequired Env.githubToken - githubCacheRef <- liftAff Cache.newCacheRef registryCacheRef <- liftAff Cache.newCacheRef let cache = Path.concat [ stateDir, "scratch", ".cache" ] - octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl debouncer <- liftAff Registry.newDebouncer - let registryEnv :: Registry.RegistryEnv registryEnv = @@ -166,7 +178,12 @@ runDailyImporterScript = do , debouncer , cacheRef: registryCacheRef } + pure { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } +-- | Run the DailyImporter script in Submit mode +runDailyImporterScript :: E2E Unit +runDailyImporterScript = do + { resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript result <- liftAff $ DailyImporter.runDailyImport DailyImporter.Submit resourceEnv.registryApiUrl # Except.runExcept @@ -175,7 +192,6 @@ runDailyImporterScript = do # Log.interpret (Log.handleTerminal Quiet) # Env.runResourceEnv resourceEnv # Run.runBaseAff' - case result of Left err -> liftAff $ Aff.throwError $ Aff.error $ "DailyImporter failed: " <> err Right _ -> pure unit @@ -183,33 +199,7 @@ runDailyImporterScript = do -- | Run the PackageTransferrer script in Submit mode runPackageTransferrerScript :: E2E Unit runPackageTransferrerScript = do - { stateDir, privateKey } <- ask - - -- Set up environment - liftEffect $ Process.chdir stateDir - - -- Get resource env from environment variables - resourceEnv <- liftEffect Env.lookupResourceEnv - token <- liftEffect $ Env.lookupRequired Env.githubToken - - githubCacheRef <- liftAff Cache.newCacheRef - registryCacheRef <- liftAff Cache.newCacheRef - let cache = Path.concat [ stateDir, "scratch", ".cache" ] - - octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl - debouncer <- liftAff Registry.newDebouncer - - let - registryEnv :: Registry.RegistryEnv - registryEnv = - { pull: Git.Autostash - , write: Registry.ReadOnly - , repos: Registry.defaultRepos - , workdir: Path.concat [ stateDir, "scratch" ] - , debouncer - , cacheRef: registryCacheRef - } - + { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript result <- liftAff $ PackageTransferrer.runPackageTransferrer PackageTransferrer.Submit (Just privateKey) resourceEnv.registryApiUrl # Except.runExcept @@ -218,7 +208,6 @@ runPackageTransferrerScript = do # Log.interpret (Log.handleTerminal Quiet) # Env.runResourceEnv resourceEnv # Run.runBaseAff' - case result of Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageTransferrer failed: " <> err Right _ -> pure unit @@ -226,33 +215,7 @@ runPackageTransferrerScript = do -- | Run the PackageSetUpdater script in Submit mode runPackageSetUpdaterScript :: E2E Unit runPackageSetUpdaterScript = do - { stateDir, privateKey } <- ask - - -- Set up environment - liftEffect $ Process.chdir stateDir - - -- Get resource env from environment variables - resourceEnv <- liftEffect Env.lookupResourceEnv - token <- liftEffect $ Env.lookupRequired Env.githubToken - - githubCacheRef <- liftAff Cache.newCacheRef - registryCacheRef <- liftAff Cache.newCacheRef - let cache = Path.concat [ stateDir, "scratch", ".cache" ] - - octokit <- liftAff $ Octokit.newOctokit token resourceEnv.githubApiUrl - debouncer <- liftAff Registry.newDebouncer - - let - registryEnv :: Registry.RegistryEnv - registryEnv = - { pull: Git.Autostash - , write: Registry.ReadOnly - , repos: Registry.defaultRepos - , workdir: Path.concat [ stateDir, "scratch" ] - , debouncer - , cacheRef: registryCacheRef - } - + { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript result <- liftAff $ PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit (Just privateKey) resourceEnv.registryApiUrl # Except.runExcept @@ -261,7 +224,6 @@ runPackageSetUpdaterScript = do # Log.interpret (Log.handleTerminal Quiet) # Env.runResourceEnv resourceEnv # Run.runBaseAff' - case result of Left err -> liftAff $ Aff.throwError $ Aff.error $ "PackageSetUpdater failed: " <> err Right _ -> pure unit From 8b6a1babe067290cb1fe48d2cb49ea709e0bc250 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sat, 17 Jan 2026 18:31:53 +0200 Subject: [PATCH 12/20] Refactor compiler version solving in the daily importer --- scripts/src/DailyImporter.purs | 136 ++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 52 deletions(-) diff --git a/scripts/src/DailyImporter.purs b/scripts/src/DailyImporter.purs index 5130e956..1239f1e2 100644 --- a/scripts/src/DailyImporter.purs +++ b/scripts/src/DailyImporter.purs @@ -21,8 +21,6 @@ import Data.Array.NonEmpty as NonEmptyArray import Data.Codec.JSON as CJ import Data.Map as Map import Data.Set as Set -import Data.Set.NonEmpty (NonEmptySet) -import Data.Set.NonEmpty as NonEmptySet import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch (Method(..)) @@ -43,11 +41,17 @@ import Registry.App.Effect.Log as Log import Registry.App.Effect.Registry (REGISTRY) import Registry.App.Effect.Registry as Registry import Registry.App.Legacy.LenientVersion as LenientVersion +import Registry.App.Legacy.Types (RawVersion(..)) +import Registry.App.Manifest.SpagoYaml as SpagoYaml +import Registry.Foreign.Octokit (Address) import Registry.Foreign.Octokit as Octokit import Registry.Location (Location(..)) +import Registry.Manifest (Manifest(..)) +import Registry.Manifest as Manifest import Registry.Operation as Operation import Registry.PackageName as PackageName import Registry.Range as Range +import Registry.Solver as Solver import Registry.Version as Version import Run (AFF, EFFECT, Run) import Run as Run @@ -121,6 +125,16 @@ runDailyImport mode registryApiUrl = do allMetadata <- Registry.readAllMetadata let packages = Map.toUnfoldable allMetadata :: Array (Tuple PackageName Metadata) + -- Build the compiler index once for all packages + Log.info "Building compiler index..." + pursVersions <- PursVersions.pursVersions + manifestIndex <- Registry.readAllManifests + let + compilerIndex = Solver.buildCompilerIndex pursVersions manifestIndex allMetadata + allCompilersRange = Range.mk + (NonEmptyArray.head pursVersions) + (Version.bumpPatch (NonEmptyArray.last pursVersions)) + Log.info $ "Checking " <> show (Array.length packages) <> " packages for new versions..." submitted <- for packages \(Tuple name (Metadata metadata)) -> do @@ -151,7 +165,7 @@ runDailyImport mode registryApiUrl = do -- Submit publish jobs for new versions count <- for newVersions \{ version, ref } -> do - submitPublishJob mode registryApiUrl allMetadata name (Metadata metadata) version ref + submitPublishJob mode registryApiUrl compilerIndex allCompilersRange name (Metadata metadata) version ref pure $ Array.length $ Array.filter identity count @@ -159,48 +173,45 @@ runDailyImport mode registryApiUrl = do Log.info $ "Daily Importer complete. Submitted " <> show totalSubmitted <> " publish jobs." -- | Submit a publish job for a new package version. --- | Attempts to find a compatible compiler by looking at the previous version's +-- | Attempts to find a compatible compiler by fetching the new version's manifest +-- | from GitHub and using the solver to find a compiler that works with its -- | dependencies. Falls back to the lowest compiler from the previous version if --- | no dependencies exist, or to the latest compiler if no previous version exists. +-- | fetching/parsing fails or the package has no dependencies, or to the latest +-- | compiler if no previous version exists. submitPublishJob :: Mode -> URL - -> Map PackageName Metadata + -> Solver.CompilerIndex + -> Maybe Range -> PackageName -> Metadata -> Version -> String -> Run DailyImportEffects Boolean -submitPublishJob mode registryApiUrl allMetadata name (Metadata metadata) version ref = do +submitPublishJob mode registryApiUrl compilerIndex allCompilersRange name (Metadata metadata) version ref = do let formatted = formatPackageVersion name version - -- Determine the appropriate compiler version - compiler <- case Map.findMax metadata.published of - Just { key: prevVersion, value: publishedInfo } -> do - -- Look up the manifest for the previous version to get its dependencies - maybeManifest <- Registry.readManifest name prevVersion - case maybeManifest of - Just (Manifest manifest) | not (Map.isEmpty manifest.dependencies) -> do - -- Use previous version's dependencies to find compatible compilers - let - depVersions :: Map PackageName Version - depVersions = Map.mapMaybeWithKey - ( \depName range -> - case Map.lookup depName allMetadata of - Just (Metadata depMeta) -> - Array.last $ Array.filter (Range.includes range) $ Array.sort $ Array.fromFoldable $ Map.keys depMeta.published - Nothing -> Nothing - ) - manifest.dependencies - - case compatibleCompilers allMetadata depVersions of - Just compilerSet -> pure $ NonEmptySet.min compilerSet - -- No intersection found, fall back to lowest compiler from previous version - Nothing -> pure $ NonEmptyArray.head publishedInfo.compilers - -- No manifest or no dependencies, fall back to lowest compiler from previous version - _ -> pure $ NonEmptyArray.head publishedInfo.compilers - Nothing -> - NonEmptyArray.last <$> PursVersions.pursVersions + -- Determine the appropriate compiler version by trying to use the new version's + -- manifest and the solver, falling back to previous version's compiler if needed. + compiler <- case metadata.location of + GitHub { owner, repo } -> do + let address = { owner, repo } + -- Try to fetch and parse the new version's dependencies from GitHub + maybeNewDeps <- fetchNewVersionDeps address ref + case maybeNewDeps of + Just deps | not (Map.isEmpty deps) -> do + -- Use solver to find a compatible compiler + case solveForCompiler compilerIndex allCompilersRange deps of + Just compilerVersion -> pure compilerVersion + Nothing -> do + Log.debug $ "Solver failed to find compiler for " <> formatted <> ", using fallback" + fallbackCompiler + _ -> do + Log.debug $ "No dependencies found for " <> formatted <> ", using fallback" + fallbackCompiler + Git _ -> + -- For non-GitHub packages, always use fallback + fallbackCompiler let payload :: Operation.PublishData @@ -228,6 +239,45 @@ submitPublishJob mode registryApiUrl allMetadata name (Metadata metadata) versio Right { jobId } -> do Log.info $ "Submitted publish job " <> unwrap jobId <> " for " <> formatted pure true + where + -- Fall back to using the previous version's lowest compiler, or latest if no previous version + fallbackCompiler :: Run DailyImportEffects Version + fallbackCompiler = case Map.findMax metadata.published of + Just { value: publishedInfo } -> pure $ NonEmptyArray.head publishedInfo.compilers + Nothing -> NonEmptyArray.last <$> PursVersions.pursVersions + +-- | Try to fetch and parse the new version's dependencies from GitHub. +-- | Tries purs.json first, then falls back to spago.yaml. +fetchNewVersionDeps :: Address -> String -> Run DailyImportEffects (Maybe (Map PackageName Range)) +fetchNewVersionDeps address ref = do + -- Try purs.json first using getJsonFile which handles JSON decoding + pursJsonResult <- GitHub.getJsonFile address (RawVersion ref) Manifest.codec "purs.json" + case pursJsonResult of + Right (Manifest manifest) -> pure $ Just manifest.dependencies + Left _ -> do + -- Fall back to spago.yaml + spagoYamlResult <- GitHub.getContent address (RawVersion ref) "spago.yaml" + case spagoYamlResult of + Right contents -> + case parseYaml SpagoYaml.spagoYamlCodec contents of + Right { package: Just pkg } -> + case SpagoYaml.convertSpagoDependencies pkg.dependencies of + Right deps -> pure $ Just deps + Left _ -> pure Nothing + _ -> pure Nothing + Left _ -> pure Nothing + +-- | Use the solver to find a compatible compiler for the given dependencies. +-- | Uses a pre-built compiler index and range covering all available compilers. +-- | Returns Nothing if no compatible compiler can be found. +solveForCompiler :: Solver.CompilerIndex -> Maybe Range -> Map PackageName Range -> Maybe Version +solveForCompiler compilerIndex allCompilersRange deps = + case allCompilersRange of + Nothing -> Nothing + Just range -> + case Solver.solveWithCompiler range compilerIndex deps of + Right (Tuple compilerVersion _) -> Just compilerVersion + Left _ -> Nothing -- | Submit a job to the registry API submitJob :: String -> Operation.PublishData -> Aff (Either String V1.JobCreatedResponse) @@ -248,21 +298,3 @@ submitJob url payload = do Right r -> pure $ Right r else pure $ Left $ "HTTP " <> show response.status <> ": " <> responseBody - --- | Given a set of package versions, determine the set of compilers that can be --- | used for all packages by intersecting their supported compiler ranges. -compatibleCompilers :: Map PackageName Metadata -> Map PackageName Version -> Maybe (NonEmptySet Version) -compatibleCompilers allMetadata resolutions = do - let - associated :: Array { compilers :: NonEmptyArray Version } - associated = Map.toUnfoldableUnordered resolutions # Array.mapMaybe \(Tuple depName depVersion) -> do - Metadata depMeta <- Map.lookup depName allMetadata - published <- Map.lookup depVersion depMeta.published - Just { compilers: published.compilers } - - case Array.uncons associated of - Nothing -> Nothing - Just { head, tail: [] } -> Just $ NonEmptySet.fromFoldable1 head.compilers - Just { head, tail } -> do - let foldFn prev = Set.intersection prev <<< Set.fromFoldable <<< _.compilers - NonEmptySet.fromFoldable $ Array.foldl foldFn (Set.fromFoldable head.compilers) tail From 84e8b8da6c704d04dd8c0bf977fa9ebed8090c66 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sat, 17 Jan 2026 22:22:09 +0200 Subject: [PATCH 13/20] More review feedback --- app-e2e/src/Test/E2E/Endpoint/Startup.purs | 4 +- app/src/App/Effect/Registry.purs | 19 ------ app/src/App/Main.purs | 21 +++---- scripts/src/DailyImporter.purs | 72 +++++++++++++++++----- 4 files changed, 68 insertions(+), 48 deletions(-) diff --git a/app-e2e/src/Test/E2E/Endpoint/Startup.purs b/app-e2e/src/Test/E2E/Endpoint/Startup.purs index 0bc03712..5f91ac30 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Startup.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Startup.purs @@ -1,4 +1,4 @@ --- | E2E tests for server startup behavior (non-scheduler). +-- | E2E tests for server startup behavior. -- | -- | IMPORTANT: These tests must run BEFORE resetTestState is called, since -- | the jobs are created at server startup and would be cleared. @@ -18,7 +18,7 @@ import Test.Spec as Spec spec :: E2ESpec spec = do - Spec.describe "checkIfNewCompiler" do + Spec.describe "check if there's a new compiler" do Spec.it "enqueues matrix jobs for packages with no dependencies when new compiler detected" do -- The test env has compilers 0.15.10 and 0.15.11 available. -- prelude@6.0.1 fixture only has compiler 0.15.10 in metadata. diff --git a/app/src/App/Effect/Registry.purs b/app/src/App/Effect/Registry.purs index da696429..48fbdf4a 100644 --- a/app/src/App/Effect/Registry.purs +++ b/app/src/App/Effect/Registry.purs @@ -34,7 +34,6 @@ import Registry.App.Legacy.PackageSet (PscTag(..)) import Registry.App.Legacy.PackageSet as Legacy.PackageSet import Registry.App.Legacy.Types (legacyPackageSetCodec) import Registry.Constants as Constants -import Registry.Foreign.FSExtra as FS.Extra import Registry.Foreign.FastGlob as FastGlob import Registry.Foreign.Octokit (Address) import Registry.Foreign.Octokit as Octokit @@ -156,24 +155,6 @@ data RepoKey | ManifestIndexRepo | LegacyPackageSetsRepo -derive instance Eq RepoKey -derive instance Ord RepoKey - --- | Validate that a repository is in a valid state. --- | If the repo is corrupted (e.g., from an interrupted clone), delete it. -validateRepo :: forall r. FilePath -> Run (LOG + AFF + EFFECT + r) Unit -validateRepo path = do - exists <- Run.liftAff $ Aff.attempt (FS.Aff.stat path) - case exists of - Left _ -> pure unit -- Doesn't exist, nothing to validate - Right _ -> do - result <- Run.liftAff $ Git.gitCLI [ "rev-parse", "--git-dir" ] (Just path) - case result of - Left _ -> do - Log.warn $ "Detected corrupted repo at " <> path <> ", deleting" - Run.liftAff $ FS.Extra.remove path - Right _ -> pure unit - -- | A legend for values that can be committed. We know where each kind of value -- | ought to exist, so we can create a correct path for any given type ourselves. data CommitKey diff --git a/app/src/App/Main.purs b/app/src/App/Main.purs index 22c873f5..8ad4fd7a 100644 --- a/app/src/App/Main.purs +++ b/app/src/App/Main.purs @@ -13,17 +13,16 @@ import Registry.App.Server.JobExecutor as JobExecutor import Registry.App.Server.Router as Router main :: Effect Unit -main = Aff.launchAff_ do - Aff.attempt createServerEnv >>= case _ of - Left error -> liftEffect do - Console.log $ "Failed to start server: " <> Aff.message error - Process.exit' 1 - Right env -> liftEffect do - case env.vars.resourceEnv.healthchecksUrl of - Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" - Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl - Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor env - Router.runRouter env +main = createServerEnv # Aff.runAff_ case _ of + Left error -> liftEffect do + Console.log $ "Failed to start server: " <> Aff.message error + Process.exit' 1 + Right env -> do + case env.vars.resourceEnv.healthchecksUrl of + Nothing -> Console.log "HEALTHCHECKS_URL not set, healthcheck pinging disabled" + Just healthchecksUrl -> Aff.launchAff_ $ healthcheck healthchecksUrl + Aff.launchAff_ $ withRetryLoop "Job executor" $ JobExecutor.runJobExecutor env + Router.runRouter env where healthcheck :: String -> Aff Unit healthcheck healthchecksUrl = loop limit diff --git a/scripts/src/DailyImporter.purs b/scripts/src/DailyImporter.purs index 1239f1e2..d8efae35 100644 --- a/scripts/src/DailyImporter.purs +++ b/scripts/src/DailyImporter.purs @@ -21,6 +21,7 @@ import Data.Array.NonEmpty as NonEmptyArray import Data.Codec.JSON as CJ import Data.Map as Map import Data.Set as Set +import Data.String as String import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch (Method(..)) @@ -41,7 +42,9 @@ import Registry.App.Effect.Log as Log import Registry.App.Effect.Registry (REGISTRY) import Registry.App.Effect.Registry as Registry import Registry.App.Legacy.LenientVersion as LenientVersion -import Registry.App.Legacy.Types (RawVersion(..)) +import Registry.App.Legacy.Manifest (Bowerfile(..), SpagoDhallJson(..)) +import Registry.App.Legacy.Manifest as Legacy.Manifest +import Registry.App.Legacy.Types (RawPackageName(..), RawVersion(..), RawVersionRange(..)) import Registry.App.Manifest.SpagoYaml as SpagoYaml import Registry.Foreign.Octokit (Address) import Registry.Foreign.Octokit as Octokit @@ -247,25 +250,62 @@ submitPublishJob mode registryApiUrl compilerIndex allCompilersRange name (Metad Nothing -> NonEmptyArray.last <$> PursVersions.pursVersions -- | Try to fetch and parse the new version's dependencies from GitHub. --- | Tries purs.json first, then falls back to spago.yaml. +-- | Tries purs.json, spago.yaml, bower.json, and spago.dhall in order. fetchNewVersionDeps :: Address -> String -> Run DailyImportEffects (Maybe (Map PackageName Range)) fetchNewVersionDeps address ref = do - -- Try purs.json first using getJsonFile which handles JSON decoding - pursJsonResult <- GitHub.getJsonFile address (RawVersion ref) Manifest.codec "purs.json" + let rawRef = RawVersion ref + -- Try purs.json first + pursJsonResult <- GitHub.getJsonFile address rawRef Manifest.codec "purs.json" case pursJsonResult of Right (Manifest manifest) -> pure $ Just manifest.dependencies - Left _ -> do - -- Fall back to spago.yaml - spagoYamlResult <- GitHub.getContent address (RawVersion ref) "spago.yaml" - case spagoYamlResult of - Right contents -> - case parseYaml SpagoYaml.spagoYamlCodec contents of - Right { package: Just pkg } -> - case SpagoYaml.convertSpagoDependencies pkg.dependencies of - Right deps -> pure $ Just deps - Left _ -> pure Nothing - _ -> pure Nothing - Left _ -> pure Nothing + Left _ -> trySpagoYaml rawRef + where + trySpagoYaml rawRef = do + spagoYamlResult <- GitHub.getContent address rawRef "spago.yaml" + case spagoYamlResult of + Right contents -> + case parseYaml SpagoYaml.spagoYamlCodec contents of + Right { package: Just pkg } -> + case SpagoYaml.convertSpagoDependencies pkg.dependencies of + Right deps -> pure $ Just deps + Left _ -> tryBowerJson rawRef + _ -> tryBowerJson rawRef + Left _ -> tryBowerJson rawRef + + tryBowerJson rawRef = do + bowerResult <- Legacy.Manifest.fetchBowerfile address rawRef + case bowerResult of + Right (Bowerfile { dependencies }) -> + -- Strip purescript- prefix and validate dependencies + let + convert = Map.mapMaybeWithKey \(RawPackageName p) range -> do + _ <- String.stripPrefix (String.Pattern "purescript-") p + pure range + in + case hush $ Legacy.Manifest.validateDependencies (convert dependencies) of + Just deps -> pure $ Just deps + Nothing -> trySpagoDhall rawRef + Left _ -> trySpagoDhall rawRef + + trySpagoDhall rawRef = do + spagoDhallResult <- Legacy.Manifest.fetchSpagoDhallJson address rawRef + case spagoDhallResult of + Right (SpagoDhallJson { dependencies, packages }) -> + -- Convert spago.dhall dependencies to ranges using the packages map + let + fixedToRange (RawVersion fixed) = do + let parsedVersion = LenientVersion.parse fixed + let bump version = Version.print (Version.bumpHighest version) + let printRange version = Array.fold [ ">=", fixed, " <", bump version ] + RawVersionRange $ either (const fixed) (printRange <<< LenientVersion.version) parsedVersion + + convert = do + let findPackage p = Map.lookup p packages + let foldFn deps p = maybe deps (\{ version } -> Map.insert p (fixedToRange version) deps) (findPackage p) + Array.foldl foldFn Map.empty dependencies + in + pure $ hush $ Legacy.Manifest.validateDependencies convert + Left _ -> pure Nothing -- | Use the solver to find a compatible compiler for the given dependencies. -- | Uses a pre-built compiler index and range covering all available compilers. From 1bf0e53db8577211856ca7380eb4f5544b256d43 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sun, 18 Jan 2026 15:51:47 +0200 Subject: [PATCH 14/20] Make compiler optional in PublishData, to simplify the DailyImport script --- app-e2e/src/Test/E2E/Endpoint/Publish.purs | 2 +- app-e2e/src/Test/E2E/Scripts.purs | 9 +- app-e2e/src/Test/E2E/Support/Fixtures.purs | 6 +- app/src/App/API.purs | 287 ++++++++++++--------- app/src/App/Server/JobExecutor.purs | 4 +- app/test/App/API.purs | 6 +- app/test/App/GitHubIssue.purs | 6 +- lib/src/Operation.purs | 4 +- scripts/src/DailyImporter.purs | 140 +--------- scripts/src/LegacyImporter.purs | 2 +- scripts/src/PackageDeleter.purs | 2 +- 11 files changed, 191 insertions(+), 277 deletions(-) diff --git a/app-e2e/src/Test/E2E/Endpoint/Publish.purs b/app-e2e/src/Test/E2E/Endpoint/Publish.purs index 47e51c95..fcb3e4e3 100644 --- a/app-e2e/src/Test/E2E/Endpoint/Publish.purs +++ b/app-e2e/src/Test/E2E/Endpoint/Publish.purs @@ -59,7 +59,7 @@ spec = do ) allJobs -- The expected compilers are: the publish compiler + all matrix job compilers - expectedCompilers = Set.fromFoldable $ Array.cons Fixtures.effectPublishData.compiler matrixCompilers + expectedCompilers = Set.fromFoldable $ maybe matrixCompilers (\c -> Array.cons c matrixCompilers) Fixtures.effectPublishData.compiler Metadata metadataAfter <- Env.readMetadata Fixtures.effect.name case Map.lookup Fixtures.effect.version metadataAfter.published of diff --git a/app-e2e/src/Test/E2E/Scripts.purs b/app-e2e/src/Test/E2E/Scripts.purs index 68cb2960..5afc75b4 100644 --- a/app-e2e/src/Test/E2E/Scripts.purs +++ b/app-e2e/src/Test/E2E/Scripts.purs @@ -70,9 +70,12 @@ spec = do case typeEqualityJob of Just (PublishJob { payload }) -> do - -- Verify compiler selection logic - when (payload.compiler /= compiler01510) do - Assert.fail $ "Expected compiler 0.15.10 but got " <> Version.print payload.compiler + -- Verify compiler is either Nothing (API will discover) or Just 0.15.10 + case payload.compiler of + Nothing -> pure unit + Just c | c /= compiler01510 -> + Assert.fail $ "Expected compiler 0.15.10 or Nothing but got " <> Version.print c + _ -> pure unit Just _ -> Assert.fail "Expected PublishJob but got different job type" Nothing -> do let publishJobs = Array.filter isPublishJob jobs diff --git a/app-e2e/src/Test/E2E/Support/Fixtures.purs b/app-e2e/src/Test/E2E/Support/Fixtures.purs index b9b4bfe2..961150b1 100644 --- a/app-e2e/src/Test/E2E/Support/Fixtures.purs +++ b/app-e2e/src/Test/E2E/Support/Fixtures.purs @@ -64,7 +64,7 @@ effectPublishData = , subdir: Nothing } , ref: "v4.0.0" - , compiler: Utils.unsafeVersion "0.15.10" + , compiler: Just $ Utils.unsafeVersion "0.15.10" , resolutions: Nothing , version: effect.version } @@ -96,7 +96,7 @@ consolePublishData = , subdir: Nothing } , ref: "v6.1.0" - , compiler: Utils.unsafeVersion "0.15.10" + , compiler: Just $ Utils.unsafeVersion "0.15.10" , resolutions: Nothing , version: console.version } @@ -112,7 +112,7 @@ unsafeCoercePublishData = , subdir: Nothing } , ref: "v6.0.0" - , compiler: Utils.unsafeVersion "0.15.10" + , compiler: Just $ Utils.unsafeVersion "0.15.10" , resolutions: Nothing , version: unsafeCoerce.version } diff --git a/app/src/App/API.purs b/app/src/App/API.purs index 8584dc27..8a857539 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -313,6 +313,68 @@ authenticated auth = case auth.payload of type PublishEffects r = (RESOURCE_ENV + PURSUIT + REGISTRY + STORAGE + SOURCE + ARCHIVE + GITHUB + COMPILER_CACHE + LEGACY_CACHE + LOG + EXCEPT String + AFF + EFFECT + r) +-- | Resolve both compiler and resolutions for a publish operation. +-- | Will come up with some sort of plan if not provided with a compiler and/or resolutions. +resolveCompilerAndDeps + :: forall r + . CompilerIndex + -> Manifest + -> Maybe Version -- payload.compiler + -> Maybe (Map PackageName Version) -- payload.resolutions + -> Run (REGISTRY + LOG + AFF + EXCEPT String + r) { compiler :: Version, resolutions :: Map PackageName Version } +resolveCompilerAndDeps compilerIndex manifest@(Manifest { dependencies }) maybeCompiler maybeResolutions = do + Log.debug "Resolving compiler and dependencies..." + case maybeCompiler of + -- if we have a compiler we can worry about the rest of the build plan + Just compiler -> do + Log.debug $ "Using provided compiler " <> Version.print compiler + resolutions <- case maybeResolutions of + -- resolutions are provided so we just check them over + Just provided -> do + validateResolutions manifest provided + pure provided + -- no resolutions, invoke the solver with the compiler + dependencies + Nothing -> + case Operation.Validation.validateDependenciesSolve compiler manifest compilerIndex of + Left errors -> Except.throw $ formatSolverErrors errors + Right resolutions -> pure resolutions + pure { compiler, resolutions } + + -- no compiler provided, we can figure it out. We only need one for publishing anyways + Nothing -> do + Log.debug "No compiler provided, solving for compiler and resolutions" + -- there is little difference for resolutions to be provided or not - we blend + -- everything in the solver anyways. The provided resolutions will mean less work + -- for the Solver since it will have a more precise range (spanning one version only) + -- to work with + let deps = maybe dependencies (map Range.exact) maybeResolutions + Tuple compiler resolutions <- do + allCompilers <- PursVersions.pursVersions + let + -- we pass in all compilers so the solver can pick one + allCompilersRange = Range.mk + (NonEmptyArray.head allCompilers) + (Version.bumpPatch (NonEmptyArray.last allCompilers)) + case allCompilersRange of + Nothing -> Except.throw "Could not construct compiler range" + Just range -> + case Solver.solveWithCompiler range compilerIndex deps of + Left errors -> Except.throw $ formatSolverErrors errors + Right result -> pure result + + Log.info $ "Discovered compiler " <> Version.print compiler + pure { compiler, resolutions } + where + formatSolverErrors errors = String.joinWith "\n" + [ "Could not produce valid dependencies for manifest." + , "```" + , errors # foldMapWithIndex \index error -> String.joinWith "\n" + [ "[Error " <> show (index + 1) <> "]" + , Solver.printSolverError error + ] + , "```" + ] + -- | Publish a package via the 'publish' operation. If the package has not been -- | published before then it will be registered and the given version will be -- | upload. If it has been published before then the existing metadata will be @@ -321,7 +383,7 @@ type PublishEffects r = (RESOURCE_ENV + PURSUIT + REGISTRY + STORAGE + SOURCE + -- The legacyIndex argument contains the unverified manifests produced by the -- legacy importer; these manifests can be used on legacy packages to conform -- them to the registry rule that transitive dependencies are not allowed. -publish :: forall r. Maybe Solver.TransitivizedRegistry -> PublishData -> Run (PublishEffects + r) (Maybe { dependencies :: Map PackageName Range, version :: Version }) +publish :: forall r. Maybe Solver.TransitivizedRegistry -> PublishData -> Run (PublishEffects + r) (Maybe { compiler :: Version, dependencies :: Map PackageName Range, version :: Version }) publish maybeLegacyIndex payload = do let printedName = PackageName.print payload.name @@ -396,30 +458,15 @@ publish maybeLegacyIndex payload = do Left err -> Except.throw $ Source.printFetchError err - Log.debug $ "Package downloaded to " <> downloadedPackage <> ", verifying it contains a src directory with valid modules..." - Internal.Path.readPursFiles (Path.concat [ downloadedPackage, "src" ]) >>= case _ of + Log.debug $ "Package downloaded to " <> downloadedPackage <> ", verifying it contains a src directory..." + srcPursFiles <- Internal.Path.readPursFiles (Path.concat [ downloadedPackage, "src" ]) >>= case _ of Nothing -> Except.throw $ Array.fold [ "This package has no PureScript files in its `src` directory. " , "All package sources must be in the `src` directory, with any additional " , "sources indicated by the `files` key in your manifest." ] - Just files -> - -- The 'validatePursModules' function uses language-cst-parser, which only - -- supports syntax back to 0.15.0. We'll still try to validate the package - -- but it may fail to parse. - Operation.Validation.validatePursModules files >>= case _ of - Left formattedError | payload.compiler < Purs.minLanguageCSTParser -> do - Log.debug $ "Package failed to parse in validatePursModules: " <> formattedError - Log.debug $ "Skipping check because package is published with a pre-0.15.0 compiler (" <> Version.print payload.compiler <> ")." - Left formattedError -> - Except.throw $ Array.fold - [ "This package has either malformed or disallowed PureScript module names " - , "in its source: " - , formattedError - ] - Right _ -> - Log.debug "Package contains well-formed .purs files in its src directory." + Just files -> pure files -- If the package doesn't have a purs.json we can try to make one - possible scenarios: -- - in case it has a spago.yaml then we know how to read that, and have all the info to move forward @@ -535,80 +582,96 @@ publish maybeLegacyIndex payload = do , "```" ] - case Operation.Validation.isNotPublished (Manifest receivedManifest) (Metadata metadata) of - -- If the package has been published already, then we check whether the published - -- version has made it to Pursuit or not. If it has, then we terminate here. If - -- it hasn't then we publish to Pursuit and then terminate. - Just info -> do - published <- Pursuit.getPublishedVersions receivedManifest.name >>= case _ of - Left error -> Except.throw error - Right versions -> pure versions - - case Map.lookup receivedManifest.version published of - Just url -> do - Except.throw $ String.joinWith "\n" - [ "You tried to upload a version that already exists: " <> Version.print receivedManifest.version - , "" - , "Its metadata is:" - , "```json" - , printJson Metadata.publishedMetadataCodec info - , "```" - , "" - , "and its documentation is available here:" - , url - ] + -- try to terminate early here: if the package is already published AND the docs + -- are on Pursuit, then we can wrap up here + for_ (Operation.Validation.isNotPublished (Manifest receivedManifest) (Metadata metadata)) \info -> do + published <- Pursuit.getPublishedVersions receivedManifest.name >>= case _ of + Left error -> Except.throw error + Right versions -> pure versions + for_ (Map.lookup receivedManifest.version published) \url -> + Except.throw $ String.joinWith "\n" + [ "You tried to upload a version that already exists: " <> Version.print receivedManifest.version + , "" + , "Its metadata is:" + , "```json" + , printJson Metadata.publishedMetadataCodec info + , "```" + , "" + , "and its documentation is available here:" + , url + ] - Nothing | payload.compiler < Purs.minPursuitPublish -> do - Log.notice $ Array.fold - [ "This version has already been published to the registry, but the docs have not been " - , "uploaded to Pursuit. Unfortunately, it is not possible to publish to Pursuit via the " - , "registry using compiler versions prior to " <> Version.print Purs.minPursuitPublish - , ". Please try with a later compiler." - ] - pure Nothing + -- Resolve compiler and resolutions. If compiler was not provided, + -- discover a compatible compiler based on dependencies. + Log.info "Verifying the package build plan..." + compilerIndex <- MatrixBuilder.readCompilerIndex + { compiler, resolutions: validatedResolutions } <- resolveCompilerAndDeps compilerIndex (Manifest receivedManifest) payload.compiler payload.resolutions + Log.info $ "Using compiler " <> Version.print compiler + + -- Validate PureScript module names now that we know the compiler. + -- language-cst-parser only supports syntax back to 0.15.0, so we skip for older compilers. + Operation.Validation.validatePursModules srcPursFiles >>= case _ of + Left formattedError | compiler < Purs.minLanguageCSTParser -> do + Log.debug $ "Package failed to parse in validatePursModules: " <> formattedError + Log.debug $ "Skipping check because package is published with a pre-0.15.0 compiler (" <> Version.print compiler <> ")." + Left formattedError -> + Except.throw $ Array.fold + [ "This package has either malformed or disallowed PureScript module names " + , "in its source: " + , formattedError + ] + Right _ -> + Log.debug "Package contains well-formed .purs files in its src directory." - Nothing -> do - Log.notice $ Array.fold - [ "This version has already been published to the registry, but the docs have not been " - , "uploaded to Pursuit. Skipping registry publishing and retrying Pursuit publishing..." - ] - compilerIndex <- MatrixBuilder.readCompilerIndex - verifiedResolutions <- verifyResolutions compilerIndex payload.compiler (Manifest receivedManifest) payload.resolutions - let installedResolutions = Path.concat [ tmp, ".registry" ] - MatrixBuilder.installBuildPlan verifiedResolutions installedResolutions - compilationResult <- Run.liftAff $ Purs.callCompiler - { command: Purs.Compile { globs: [ "src/**/*.purs", Path.concat [ installedResolutions, "*/src/**/*.purs" ] ] } - , version: Just payload.compiler - , cwd: Just downloadedPackage - } - case compilationResult of - Left compileFailure -> do - let error = MatrixBuilder.printCompilerFailure payload.compiler compileFailure - Log.error $ "Compilation failed, cannot upload to pursuit: " <> error - Except.throw "Cannot publish to Pursuit because this package failed to compile." + case Operation.Validation.isNotPublished (Manifest receivedManifest) (Metadata metadata) of + -- If the package has been published already but docs for this version are missing + -- from Pursuit (we check earlier if the docs are there, so we end up here if they are not) + -- then upload to Pursuit and terminate + Just _ | compiler < Purs.minPursuitPublish -> do + Log.notice $ Array.fold + [ "This version has already been published to the registry, but the docs have not been " + , "uploaded to Pursuit. Unfortunately, it is not possible to publish to Pursuit via the " + , "registry using compiler versions prior to " <> Version.print Purs.minPursuitPublish + , ". Please try with a later compiler." + ] + pure Nothing + + Just _ -> do + Log.notice $ Array.fold + [ "This version has already been published to the registry, but the docs have not been " + , "uploaded to Pursuit. Skipping registry publishing and retrying Pursuit publishing..." + ] + let installedResolutions = Path.concat [ tmp, ".registry" ] + MatrixBuilder.installBuildPlan validatedResolutions installedResolutions + compilationResult <- Run.liftAff $ Purs.callCompiler + { command: Purs.Compile { globs: [ "src/**/*.purs", Path.concat [ installedResolutions, "*/src/**/*.purs" ] ] } + , version: Just compiler + , cwd: Just downloadedPackage + } + case compilationResult of + Left compileFailure -> do + let error = MatrixBuilder.printCompilerFailure compiler compileFailure + Log.error $ "Compilation failed, cannot upload to pursuit: " <> error + Except.throw "Cannot publish to Pursuit because this package failed to compile." + Right _ -> do + Log.debug "Uploading to Pursuit" + -- While we have created a manifest from the package source, we + -- still need to ensure a purs.json file exists for 'purs publish'. + unless hadPursJson do + existingManifest <- ManifestIndex.readManifest receivedManifest.name receivedManifest.version + case existingManifest of + Nothing -> Except.throw "Version was previously published, but we could not find a purs.json file in the package source, and no existing manifest was found in the registry." + Just existing -> Run.liftAff $ writeJsonFile Manifest.codec packagePursJson existing + publishToPursuit { source: downloadedPackage, compiler, resolutions: validatedResolutions, installedResolutions } >>= case _ of + Left publishErr -> Except.throw publishErr Right _ -> do - Log.debug "Uploading to Pursuit" - -- While we have created a manifest from the package source, we - -- still need to ensure a purs.json file exists for 'purs publish'. - unless hadPursJson do - existingManifest <- ManifestIndex.readManifest receivedManifest.name receivedManifest.version - case existingManifest of - Nothing -> Except.throw "Version was previously published, but we could not find a purs.json file in the package source, and no existing manifest was found in the registry." - Just existing -> Run.liftAff $ writeJsonFile Manifest.codec packagePursJson existing - publishToPursuit { source: downloadedPackage, compiler: payload.compiler, resolutions: verifiedResolutions, installedResolutions } >>= case _ of - Left publishErr -> Except.throw publishErr - Right _ -> do - FS.Extra.remove tmp - Log.notice "Successfully uploaded package docs to Pursuit! 🎉 🚀" - pure Nothing + FS.Extra.remove tmp + Log.notice "Successfully uploaded package docs to Pursuit! 🎉 🚀" + pure Nothing -- In this case the package version has not been published, so we proceed -- with ordinary publishing. Nothing -> do - Log.info "Verifying the package build plan..." - compilerIndex <- MatrixBuilder.readCompilerIndex - validatedResolutions <- verifyResolutions compilerIndex payload.compiler (Manifest receivedManifest) payload.resolutions - Log.notice "Verifying unused and/or missing dependencies..." -- First we install the resolutions and call 'purs graph' to adjust the @@ -622,7 +685,7 @@ publish maybeLegacyIndex payload = do let pursGraph = Purs.Graph { globs: [ srcGlobs, depGlobs ] } -- We need to use the minimum compiler version that supports 'purs graph'. - let pursGraphCompiler = if payload.compiler >= Purs.minPursGraph then payload.compiler else Purs.minPursGraph + let pursGraphCompiler = if compiler >= Purs.minPursGraph then compiler else Purs.minPursGraph -- In this step we run 'purs graph' to get a graph of the package source -- and installed dependencies and use that to determine if the manifest @@ -676,7 +739,7 @@ publish maybeLegacyIndex payload = do Except.throw $ "Failed to validate unused / missing dependencies: " <> Operation.Validation.printValidateDepsError depError Just legacyIndex -> do Log.info $ "Found fixable dependency errors: " <> Operation.Validation.printValidateDepsError depError - conformLegacyManifest (Manifest receivedManifest) payload.compiler compilerIndex legacyIndex depError + conformLegacyManifest (Manifest receivedManifest) compiler compilerIndex legacyIndex depError -- If the check passes then we can simply return the manifest and -- resolutions. @@ -700,7 +763,7 @@ publish maybeLegacyIndex payload = do -- the package with exactly what is going to be uploaded. Log.notice $ Array.fold [ "Verifying package compiles using compiler " - , Version.print payload.compiler + , Version.print compiler , " and resolutions:\n" , "```json\n" , printJson (Internal.Codec.packageMap Version.codec) resolutions @@ -713,13 +776,13 @@ publish maybeLegacyIndex payload = do MatrixBuilder.installBuildPlan resolutions installedResolutions compilationResult <- Run.liftAff $ Purs.callCompiler { command: Purs.Compile { globs: [ Path.concat [ packageSource, "src/**/*.purs" ], Path.concat [ installedResolutions, "*/src/**/*.purs" ] ] } - , version: Just payload.compiler + , version: Just compiler , cwd: Just tmp } case compilationResult of Left compileFailure -> do - let error = MatrixBuilder.printCompilerFailure payload.compiler compileFailure + let error = MatrixBuilder.printCompilerFailure compiler compileFailure Except.throw $ "Publishing failed due to a compiler error:\n\n" <> error Right _ -> pure unit @@ -749,7 +812,7 @@ publish maybeLegacyIndex payload = do Storage.upload (un Manifest manifest).name (un Manifest manifest).version tarballPath Log.debug $ "Adding the new version " <> Version.print (un Manifest manifest).version <> " to the package metadata file." - let newPublishedVersion = { hash, compilers: NonEmptyArray.singleton payload.compiler, publishedTime, bytes } + let newPublishedVersion = { hash, compilers: NonEmptyArray.singleton compiler, publishedTime, bytes } let newMetadata = metadata { published = Map.insert (un Manifest manifest).version newPublishedVersion metadata.published } Registry.writeMetadata (un Manifest manifest).name (Metadata newMetadata) @@ -764,11 +827,11 @@ publish maybeLegacyIndex payload = do Log.notice "Mirrored registry operation to the legacy registry!" Log.debug "Uploading package documentation to Pursuit" - if payload.compiler >= Purs.minPursuitPublish then + if compiler >= Purs.minPursuitPublish then -- TODO: We must use the 'downloadedPackage' instead of 'packageSource' -- because Pursuit requires a git repository, and our tarball directory -- is not one. This should be changed once Pursuit no longer needs git. - publishToPursuit { source: downloadedPackage, compiler: payload.compiler, resolutions, installedResolutions } >>= case _ of + publishToPursuit { source: downloadedPackage, compiler, resolutions, installedResolutions } >>= case _ of Left publishErr -> do Log.error publishErr Log.notice $ "Failed to publish package docs to Pursuit: " <> publishErr @@ -777,7 +840,7 @@ publish maybeLegacyIndex payload = do else do Log.notice $ Array.fold [ "Skipping Pursuit publishing because this package was published with a pre-0.14.7 compiler (" - , Version.print payload.compiler + , Version.print compiler , "). If you want to publish documentation, please try again with a later compiler." ] @@ -787,15 +850,15 @@ publish maybeLegacyIndex payload = do for_ maybeLegacyIndex \_idx -> do Log.notice "Determining all valid compiler versions for this package..." allCompilers <- PursVersions.pursVersions - { failed: invalidCompilers, succeeded: validCompilers } <- case NonEmptyArray.fromFoldable $ NonEmptyArray.delete payload.compiler allCompilers of - Nothing -> pure { failed: Map.empty, succeeded: NonEmptySet.singleton payload.compiler } + { failed: invalidCompilers, succeeded: validCompilers } <- case NonEmptyArray.fromFoldable $ NonEmptyArray.delete compiler allCompilers of + Nothing -> pure { failed: Map.empty, succeeded: NonEmptySet.singleton compiler } Just try -> do found <- findAllCompilers { source: packageSource , manifest , compilers: try } - pure { failed: found.failed, succeeded: NonEmptySet.cons payload.compiler found.succeeded } + pure { failed: found.failed, succeeded: NonEmptySet.cons compiler found.succeeded } unless (Map.isEmpty invalidCompilers) do Log.debug $ "Some compilers failed: " <> String.joinWith ", " (map Version.print (Set.toUnfoldable (Map.keys invalidCompilers))) @@ -809,33 +872,7 @@ publish maybeLegacyIndex payload = do Log.notice "Wrote completed metadata to the registry!" FS.Extra.remove tmp - pure $ Just { dependencies: (un Manifest manifest).dependencies, version: (un Manifest manifest).version } - --- | Verify the build plan for the package. If the user provided a build plan, --- | we ensure that the provided versions are within the ranges listed in the --- | manifest. If not, we solve their manifest to produce a build plan. -verifyResolutions :: forall r. CompilerIndex -> Version -> Manifest -> Maybe (Map PackageName Version) -> Run (REGISTRY + LOG + AFF + EXCEPT String + r) (Map PackageName Version) -verifyResolutions compilerIndex compiler manifest resolutions = do - Log.debug "Check the submitted build plan matches the manifest" - case resolutions of - Nothing -> do - case Operation.Validation.validateDependenciesSolve compiler manifest compilerIndex of - Left errors -> do - let - printedError = String.joinWith "\n" - [ "Could not produce valid dependencies for manifest." - , "```" - , errors # foldMapWithIndex \index error -> String.joinWith "\n" - [ "[Error " <> show (index + 1) <> "]" - , Solver.printSolverError error - ] - , "```" - ] - Except.throw printedError - Right solved -> pure solved - Just provided -> do - validateResolutions manifest provided - pure provided + pure $ Just { compiler, dependencies: (un Manifest manifest).dependencies, version: (un Manifest manifest).version } validateResolutions :: forall r. Manifest -> Map PackageName Version -> Run (EXCEPT String + r) Unit validateResolutions manifest resolutions = do diff --git a/app/src/App/Server/JobExecutor.purs b/app/src/App/Server/JobExecutor.purs index 4970fa93..3963849a 100644 --- a/app/src/App/Server/JobExecutor.purs +++ b/app/src/App/Server/JobExecutor.purs @@ -106,13 +106,13 @@ executeJob _ = case _ of maybeResult <- API.publish Nothing payload -- The above operation will throw if not successful, and return a map of -- dependencies of the package only if it has not been published before. - for_ maybeResult \{ dependencies, version } -> do + for_ maybeResult \{ compiler, dependencies, version } -> do -- At this point this package has been verified with one compiler only. -- So we need to enqueue compilation jobs for (1) same package, all the other -- compilers, and (2) same compiler, all packages that depend on this one -- TODO here we are building the compiler index, but we should really cache it compilerIndex <- MatrixBuilder.readCompilerIndex - let solverData = { compiler: payload.compiler, name, version, dependencies, compilerIndex } + let solverData = { compiler, name, version, dependencies, compilerIndex } samePackageAllCompilers <- MatrixBuilder.solveForAllCompilers solverData sameCompilerAllDependants <- MatrixBuilder.solveDependantsForCompiler solverData for (Array.fromFoldable $ Set.union samePackageAllCompilers sameCompilerAllDependants) diff --git a/app/test/App/API.purs b/app/test/App/API.purs index 78c02e3e..1eba0125 100644 --- a/app/test/App/API.purs +++ b/app/test/App/API.purs @@ -94,7 +94,7 @@ spec = do version = Utils.unsafeVersion "4.0.0" ref = "v4.0.0" publishArgs = - { compiler: Utils.unsafeVersion "0.15.10" + { compiler: Just $ Utils.unsafeVersion "0.15.10" , location: Just $ GitHub { owner: "purescript", repo: "purescript-effect", subdir: Nothing } , name , ref @@ -159,7 +159,7 @@ spec = do -- but did not have documentation make it to Pursuit. let pursuitOnlyPublishArgs = - { compiler: Utils.unsafeVersion "0.15.10" + { compiler: Just $ Utils.unsafeVersion "0.15.10" , location: Just $ GitHub { owner: "purescript", repo: "purescript-type-equality", subdir: Nothing } , name: Utils.unsafePackageName "type-equality" , ref: "v4.0.1" @@ -174,7 +174,7 @@ spec = do let transitive = { name: Utils.unsafePackageName "transitive", version: Utils.unsafeVersion "1.0.0" } transitivePublishArgs = - { compiler: Utils.unsafeVersion "0.15.10" + { compiler: Just $ Utils.unsafeVersion "0.15.10" , location: Just $ GitHub { owner: "purescript", repo: "purescript-transitive", subdir: Nothing } , name: transitive.name , ref: "v" <> Version.print transitive.version diff --git a/app/test/App/GitHubIssue.purs b/app/test/App/GitHubIssue.purs index d2c6baf1..6aaa6b3c 100644 --- a/app/test/App/GitHubIssue.purs +++ b/app/test/App/GitHubIssue.purs @@ -33,7 +33,7 @@ decodeEventsToOps = do { name: Utils.unsafePackageName "something" , ref: "v1.2.3" , version: Utils.unsafeVersion "1.2.3" - , compiler: Utils.unsafeVersion "0.15.0" + , compiler: Just $ Utils.unsafeVersion "0.15.0" , resolutions: Just $ Map.fromFoldable [ Utils.unsafePackageName "prelude" /\ Utils.unsafeVersion "1.0.0" ] , location: Nothing } @@ -50,7 +50,7 @@ decodeEventsToOps = do , ref: "v5.0.0" , version: Utils.unsafeVersion "5.0.0" , location: Just $ GitHub { subdir: Nothing, owner: "purescript", repo: "purescript-prelude" } - , compiler: Utils.unsafeVersion "0.15.0" + , compiler: Just $ Utils.unsafeVersion "0.15.0" , resolutions: Just $ Map.fromFoldable [ Utils.unsafePackageName "prelude" /\ Utils.unsafeVersion "1.0.0" ] } @@ -79,7 +79,7 @@ decodeEventsToOps = do , ref: "v5.0.0" , version: Utils.unsafeVersion "5.0.0" , location: Just $ GitHub { subdir: Nothing, owner: "purescript", repo: "purescript-prelude" } - , compiler: Utils.unsafeVersion "0.15.0" + , compiler: Just $ Utils.unsafeVersion "0.15.0" , resolutions: Nothing } diff --git a/lib/src/Operation.purs b/lib/src/Operation.purs index 7327001e..83debc9c 100644 --- a/lib/src/Operation.purs +++ b/lib/src/Operation.purs @@ -99,7 +99,7 @@ type PublishData = , location :: Maybe Location , ref :: String , version :: Version - , compiler :: Version + , compiler :: Maybe Version , resolutions :: Maybe (Map PackageName Version) } @@ -110,7 +110,7 @@ publishCodec = CJ.named "Publish" $ CJ.Record.object , location: CJ.Record.optional Location.codec , ref: CJ.string , version: Version.codec - , compiler: Version.codec + , compiler: CJ.Record.optional Version.codec , resolutions: CJ.Record.optional (Internal.Codec.packageMap Version.codec) } diff --git a/scripts/src/DailyImporter.purs b/scripts/src/DailyImporter.purs index d8efae35..fe4c095f 100644 --- a/scripts/src/DailyImporter.purs +++ b/scripts/src/DailyImporter.purs @@ -17,11 +17,9 @@ import ArgParse.Basic (ArgParser) import ArgParse.Basic as Arg import Codec.JSON.DecodeError as CJ.DecodeError import Data.Array as Array -import Data.Array.NonEmpty as NonEmptyArray import Data.Codec.JSON as CJ import Data.Map as Map import Data.Set as Set -import Data.String as String import Effect.Aff as Aff import Effect.Class.Console as Console import Fetch (Method(..)) @@ -31,7 +29,6 @@ import Node.Path as Path import Node.Process as Process import Registry.API.V1 as V1 import Registry.App.CLI.Git as Git -import Registry.App.CLI.PursVersions as PursVersions import Registry.App.Effect.Cache as Cache import Registry.App.Effect.Env (RESOURCE_ENV) import Registry.App.Effect.Env as Env @@ -42,19 +39,10 @@ import Registry.App.Effect.Log as Log import Registry.App.Effect.Registry (REGISTRY) import Registry.App.Effect.Registry as Registry import Registry.App.Legacy.LenientVersion as LenientVersion -import Registry.App.Legacy.Manifest (Bowerfile(..), SpagoDhallJson(..)) -import Registry.App.Legacy.Manifest as Legacy.Manifest -import Registry.App.Legacy.Types (RawPackageName(..), RawVersion(..), RawVersionRange(..)) -import Registry.App.Manifest.SpagoYaml as SpagoYaml -import Registry.Foreign.Octokit (Address) import Registry.Foreign.Octokit as Octokit import Registry.Location (Location(..)) -import Registry.Manifest (Manifest(..)) -import Registry.Manifest as Manifest import Registry.Operation as Operation import Registry.PackageName as PackageName -import Registry.Range as Range -import Registry.Solver as Solver import Registry.Version as Version import Run (AFF, EFFECT, Run) import Run as Run @@ -128,16 +116,6 @@ runDailyImport mode registryApiUrl = do allMetadata <- Registry.readAllMetadata let packages = Map.toUnfoldable allMetadata :: Array (Tuple PackageName Metadata) - -- Build the compiler index once for all packages - Log.info "Building compiler index..." - pursVersions <- PursVersions.pursVersions - manifestIndex <- Registry.readAllManifests - let - compilerIndex = Solver.buildCompilerIndex pursVersions manifestIndex allMetadata - allCompilersRange = Range.mk - (NonEmptyArray.head pursVersions) - (Version.bumpPatch (NonEmptyArray.last pursVersions)) - Log.info $ "Checking " <> show (Array.length packages) <> " packages for new versions..." submitted <- for packages \(Tuple name (Metadata metadata)) -> do @@ -168,7 +146,7 @@ runDailyImport mode registryApiUrl = do -- Submit publish jobs for new versions count <- for newVersions \{ version, ref } -> do - submitPublishJob mode registryApiUrl compilerIndex allCompilersRange name (Metadata metadata) version ref + submitPublishJob mode registryApiUrl name version ref pure $ Array.length $ Array.filter identity count @@ -176,46 +154,18 @@ runDailyImport mode registryApiUrl = do Log.info $ "Daily Importer complete. Submitted " <> show totalSubmitted <> " publish jobs." -- | Submit a publish job for a new package version. --- | Attempts to find a compatible compiler by fetching the new version's manifest --- | from GitHub and using the solver to find a compiler that works with its --- | dependencies. Falls back to the lowest compiler from the previous version if --- | fetching/parsing fails or the package has no dependencies, or to the latest --- | compiler if no previous version exists. +-- | The compiler is not specified; the registry API will discover the oldest +-- | compatible compiler based on the package's dependencies. submitPublishJob :: Mode -> URL - -> Solver.CompilerIndex - -> Maybe Range -> PackageName - -> Metadata -> Version -> String -> Run DailyImportEffects Boolean -submitPublishJob mode registryApiUrl compilerIndex allCompilersRange name (Metadata metadata) version ref = do +submitPublishJob mode registryApiUrl name version ref = do let formatted = formatPackageVersion name version - -- Determine the appropriate compiler version by trying to use the new version's - -- manifest and the solver, falling back to previous version's compiler if needed. - compiler <- case metadata.location of - GitHub { owner, repo } -> do - let address = { owner, repo } - -- Try to fetch and parse the new version's dependencies from GitHub - maybeNewDeps <- fetchNewVersionDeps address ref - case maybeNewDeps of - Just deps | not (Map.isEmpty deps) -> do - -- Use solver to find a compatible compiler - case solveForCompiler compilerIndex allCompilersRange deps of - Just compilerVersion -> pure compilerVersion - Nothing -> do - Log.debug $ "Solver failed to find compiler for " <> formatted <> ", using fallback" - fallbackCompiler - _ -> do - Log.debug $ "No dependencies found for " <> formatted <> ", using fallback" - fallbackCompiler - Git _ -> - -- For non-GitHub packages, always use fallback - fallbackCompiler - let payload :: Operation.PublishData payload = @@ -223,17 +173,17 @@ submitPublishJob mode registryApiUrl compilerIndex allCompilersRange name (Metad , version , location: Nothing -- Use current metadata location at publish time , ref - , compiler + , compiler: Nothing -- Let the API discover the oldest compatible compiler , resolutions: Nothing } case mode of DryRun -> do - Log.info $ "[DRY RUN] Would submit publish job for " <> formatted <> " with compiler " <> Version.print compiler + Log.info $ "[DRY RUN] Would submit publish job for " <> formatted pure true Submit -> do - Log.info $ "Submitting publish job for " <> formatted <> " with compiler " <> Version.print compiler + Log.info $ "Submitting publish job for " <> formatted result <- Run.liftAff $ submitJob (registryApiUrl <> "/v1/publish") payload case result of Left err -> do @@ -242,82 +192,6 @@ submitPublishJob mode registryApiUrl compilerIndex allCompilersRange name (Metad Right { jobId } -> do Log.info $ "Submitted publish job " <> unwrap jobId <> " for " <> formatted pure true - where - -- Fall back to using the previous version's lowest compiler, or latest if no previous version - fallbackCompiler :: Run DailyImportEffects Version - fallbackCompiler = case Map.findMax metadata.published of - Just { value: publishedInfo } -> pure $ NonEmptyArray.head publishedInfo.compilers - Nothing -> NonEmptyArray.last <$> PursVersions.pursVersions - --- | Try to fetch and parse the new version's dependencies from GitHub. --- | Tries purs.json, spago.yaml, bower.json, and spago.dhall in order. -fetchNewVersionDeps :: Address -> String -> Run DailyImportEffects (Maybe (Map PackageName Range)) -fetchNewVersionDeps address ref = do - let rawRef = RawVersion ref - -- Try purs.json first - pursJsonResult <- GitHub.getJsonFile address rawRef Manifest.codec "purs.json" - case pursJsonResult of - Right (Manifest manifest) -> pure $ Just manifest.dependencies - Left _ -> trySpagoYaml rawRef - where - trySpagoYaml rawRef = do - spagoYamlResult <- GitHub.getContent address rawRef "spago.yaml" - case spagoYamlResult of - Right contents -> - case parseYaml SpagoYaml.spagoYamlCodec contents of - Right { package: Just pkg } -> - case SpagoYaml.convertSpagoDependencies pkg.dependencies of - Right deps -> pure $ Just deps - Left _ -> tryBowerJson rawRef - _ -> tryBowerJson rawRef - Left _ -> tryBowerJson rawRef - - tryBowerJson rawRef = do - bowerResult <- Legacy.Manifest.fetchBowerfile address rawRef - case bowerResult of - Right (Bowerfile { dependencies }) -> - -- Strip purescript- prefix and validate dependencies - let - convert = Map.mapMaybeWithKey \(RawPackageName p) range -> do - _ <- String.stripPrefix (String.Pattern "purescript-") p - pure range - in - case hush $ Legacy.Manifest.validateDependencies (convert dependencies) of - Just deps -> pure $ Just deps - Nothing -> trySpagoDhall rawRef - Left _ -> trySpagoDhall rawRef - - trySpagoDhall rawRef = do - spagoDhallResult <- Legacy.Manifest.fetchSpagoDhallJson address rawRef - case spagoDhallResult of - Right (SpagoDhallJson { dependencies, packages }) -> - -- Convert spago.dhall dependencies to ranges using the packages map - let - fixedToRange (RawVersion fixed) = do - let parsedVersion = LenientVersion.parse fixed - let bump version = Version.print (Version.bumpHighest version) - let printRange version = Array.fold [ ">=", fixed, " <", bump version ] - RawVersionRange $ either (const fixed) (printRange <<< LenientVersion.version) parsedVersion - - convert = do - let findPackage p = Map.lookup p packages - let foldFn deps p = maybe deps (\{ version } -> Map.insert p (fixedToRange version) deps) (findPackage p) - Array.foldl foldFn Map.empty dependencies - in - pure $ hush $ Legacy.Manifest.validateDependencies convert - Left _ -> pure Nothing - --- | Use the solver to find a compatible compiler for the given dependencies. --- | Uses a pre-built compiler index and range covering all available compilers. --- | Returns Nothing if no compatible compiler can be found. -solveForCompiler :: Solver.CompilerIndex -> Maybe Range -> Map PackageName Range -> Maybe Version -solveForCompiler compilerIndex allCompilersRange deps = - case allCompilersRange of - Nothing -> Nothing - Just range -> - case Solver.solveWithCompiler range compilerIndex deps of - Right (Tuple compilerVersion _) -> Just compilerVersion - Left _ -> Nothing -- | Submit a job to the registry API submitJob :: String -> Operation.PublishData -> Aff (Either String V1.JobCreatedResponse) diff --git a/scripts/src/LegacyImporter.purs b/scripts/src/LegacyImporter.purs index 7e65c48c..2d575c72 100644 --- a/scripts/src/LegacyImporter.purs +++ b/scripts/src/LegacyImporter.purs @@ -544,7 +544,7 @@ runLegacyImport logs = do , location: Just manifest.location , ref , version: manifest.version - , compiler + , compiler: Just compiler , resolutions: Just resolutions } Run.Except.runExcept (API.publish (Just legacyIndex) payload) >>= case _ of diff --git a/scripts/src/PackageDeleter.purs b/scripts/src/PackageDeleter.purs index 257a7b1a..61bfca71 100644 --- a/scripts/src/PackageDeleter.purs +++ b/scripts/src/PackageDeleter.purs @@ -247,6 +247,6 @@ deleteVersion arguments name version = do , name: name , ref: manifest.ref , version: version - , compiler: unsafeFromRight $ Version.parse "0.15.4" + , compiler: Just $ unsafeFromRight $ Version.parse "0.15.4" , resolutions: Nothing } From 8792802dd0d734eb4ea535ed5f26f226b06a1d96 Mon Sep 17 00:00:00 2001 From: Fabrizio Ferrai Date: Sun, 18 Jan 2026 16:30:55 +0200 Subject: [PATCH 15/20] Remove signing from the package sets updater --- app-e2e/src/Test/E2E/Scripts.purs | 4 ++-- scripts/src/DailyImporter.purs | 1 - scripts/src/PackageSetUpdater.purs | 26 +++++--------------------- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/app-e2e/src/Test/E2E/Scripts.purs b/app-e2e/src/Test/E2E/Scripts.purs index 5afc75b4..cf55ce6d 100644 --- a/app-e2e/src/Test/E2E/Scripts.purs +++ b/app-e2e/src/Test/E2E/Scripts.purs @@ -218,9 +218,9 @@ runPackageTransferrerScript = do -- | Run the PackageSetUpdater script in Submit mode runPackageSetUpdaterScript :: E2E Unit runPackageSetUpdaterScript = do - { privateKey, resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript + { resourceEnv, registryEnv, octokit, cache, githubCacheRef } <- setupScript result <- liftAff - $ PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit (Just privateKey) resourceEnv.registryApiUrl + $ PackageSetUpdater.runPackageSetUpdater PackageSetUpdater.Submit resourceEnv.registryApiUrl # Except.runExcept # Registry.interpret (Registry.handle registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) diff --git a/scripts/src/DailyImporter.purs b/scripts/src/DailyImporter.purs index fe4c095f..5d07aad0 100644 --- a/scripts/src/DailyImporter.purs +++ b/scripts/src/DailyImporter.purs @@ -43,7 +43,6 @@ import Registry.Foreign.Octokit as Octokit import Registry.Location (Location(..)) import Registry.Operation as Operation import Registry.PackageName as PackageName -import Registry.Version as Version import Run (AFF, EFFECT, Run) import Run as Run import Run.Except (EXCEPT) diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs index 3b3f9aee..aa0eae4d 100644 --- a/scripts/src/PackageSetUpdater.purs +++ b/scripts/src/PackageSetUpdater.purs @@ -7,7 +7,6 @@ -- | -- | Required environment variables: -- | GITHUB_TOKEN - GitHub API token --- | PACCHETTIBOTTI_ED25519 - Private key for signing (only for --submit) -- | REGISTRY_API_URL - Registry API URL (default: https://registry.purescript.org) module Registry.Scripts.PackageSetUpdater where @@ -29,7 +28,6 @@ import JSON as JSON import Node.Path as Path import Node.Process as Process import Registry.API.V1 as V1 -import Registry.App.Auth as Auth import Registry.App.CLI.Git as Git import Registry.App.Effect.Cache as Cache import Registry.App.Effect.Env (RESOURCE_ENV) @@ -79,11 +77,6 @@ main = launchAff_ do resourceEnv <- Env.lookupResourceEnv token <- Env.lookupRequired Env.githubToken - -- Only require pacchettibotti keys in submit mode - maybePrivateKey <- case mode of - DryRun -> pure Nothing - Submit -> Just <$> Env.lookupRequired Env.pacchettibottiED25519 - githubCacheRef <- Cache.newCacheRef registryCacheRef <- Cache.newCacheRef let cache = Path.concat [ scratchDir, ".cache" ] @@ -102,7 +95,7 @@ main = launchAff_ do , cacheRef: registryCacheRef } - runPackageSetUpdater mode maybePrivateKey resourceEnv.registryApiUrl + runPackageSetUpdater mode resourceEnv.registryApiUrl # Except.runExcept # Registry.interpret (Registry.handle registryEnv) # GitHub.interpret (GitHub.handle { octokit, cache, ref: githubCacheRef }) @@ -117,8 +110,8 @@ main = launchAff_ do type PackageSetUpdaterEffects = (REGISTRY + GITHUB + LOG + RESOURCE_ENV + EXCEPT String + AFF + EFFECT + ()) -runPackageSetUpdater :: Mode -> Maybe String -> URL -> Run PackageSetUpdaterEffects Unit -runPackageSetUpdater mode maybePrivateKey registryApiUrl = do +runPackageSetUpdater :: Mode -> URL -> Run PackageSetUpdaterEffects Unit +runPackageSetUpdater mode registryApiUrl = do Log.info "Package Set Updater: checking for recent uploads..." -- Get the current package set @@ -175,22 +168,13 @@ runPackageSetUpdater mode maybePrivateKey registryApiUrl = do Log.info $ " - " <> PackageName.print name <> "@" <> Version.print version Submit -> do - privateKey <- case maybePrivateKey of - Nothing -> Except.throw "PACCHETTIBOTTI_ED25519 required for --submit mode" - Just pk -> pure pk - - -- Sign the payload with pacchettibotti keys - let rawPayload = JSON.print $ CJ.encode Operation.packageSetUpdateCodec payload - signature <- case Auth.signPayload { privateKey, rawPayload } of - Left err -> Except.throw $ "Error signing package set update: " <> err - Right sig -> pure sig - let + rawPayload = JSON.print $ CJ.encode Operation.packageSetUpdateCodec payload request :: Operation.PackageSetUpdateRequest request = { payload: PackageSetUpdate payload , rawPayload - , signature: Just signature + , signature: Nothing } Log.info $ "Submitting package set update..." From 5148c90564f4a40fe72b03f324bee8e348a6080d Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sun, 18 Jan 2026 10:08:59 -0500 Subject: [PATCH 16/20] tweaks --- app/src/App/API.purs | 6 ++---- scripts/src/PackageSetUpdater.purs | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/src/App/API.purs b/app/src/App/API.purs index 8a857539..44588cc3 100644 --- a/app/src/App/API.purs +++ b/app/src/App/API.purs @@ -343,10 +343,8 @@ resolveCompilerAndDeps compilerIndex manifest@(Manifest { dependencies }) maybeC -- no compiler provided, we can figure it out. We only need one for publishing anyways Nothing -> do Log.debug "No compiler provided, solving for compiler and resolutions" - -- there is little difference for resolutions to be provided or not - we blend - -- everything in the solver anyways. The provided resolutions will mean less work - -- for the Solver since it will have a more precise range (spanning one version only) - -- to work with + -- If resolutions are provided, validate them against the manifest first + for_ maybeResolutions \provided -> validateResolutions manifest provided let deps = maybe dependencies (map Range.exact) maybeResolutions Tuple compiler resolutions <- do allCompilers <- PursVersions.pursVersions diff --git a/scripts/src/PackageSetUpdater.purs b/scripts/src/PackageSetUpdater.purs index aa0eae4d..8113a495 100644 --- a/scripts/src/PackageSetUpdater.purs +++ b/scripts/src/PackageSetUpdater.purs @@ -170,6 +170,7 @@ runPackageSetUpdater mode registryApiUrl = do Submit -> do let rawPayload = JSON.print $ CJ.encode Operation.packageSetUpdateCodec payload + request :: Operation.PackageSetUpdateRequest request = { payload: PackageSetUpdate payload From 848452be2c4cb987f46f54856577c73200e8c108 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sun, 18 Jan 2026 10:16:49 -0500 Subject: [PATCH 17/20] fix tests --- nix/test/integration.nix | 3 ++- nix/test/test-env.nix | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nix/test/integration.nix b/nix/test/integration.nix index 75b6e648..2a374348 100644 --- a/nix/test/integration.nix +++ b/nix/test/integration.nix @@ -39,7 +39,7 @@ else pkgs.nodejs pkgs.curl pkgs.jq - pkgs.git + testSupport.gitMock pkgs.sqlite pkgs.nss_wrapper testSupport.wiremockStartScript @@ -61,6 +61,7 @@ else export HOME=$TMPDIR export STATE_DIR=$TMPDIR/state export REPO_FIXTURES_DIR="$STATE_DIR/repo-fixtures" + export GIT_BINARY="${pkgs.git}/bin/git" # Export test environment variables for E2E test runners ${testSupport.envToExports testSupport.testEnv} diff --git a/nix/test/test-env.nix b/nix/test/test-env.nix index 764d01c4..8d854031 100644 --- a/nix/test/test-env.nix +++ b/nix/test/test-env.nix @@ -129,6 +129,7 @@ in setupGitFixtures testEnv envToExports + gitMock ; # Full testConfig still available for less common access patterns From 4a7ce6da7b9e1bf1b636e8ddac70c0553630727c Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sun, 18 Jan 2026 10:33:27 -0500 Subject: [PATCH 18/20] clean up test handling --- flake.nix | 4 +--- nix/test/config.nix | 37 +++++++++++++++++++++++++++---------- nix/test/integration.nix | 12 ++++-------- nix/test/test-env.nix | 15 +++++++-------- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/flake.nix b/flake.nix index 68dfe307..55ef5299 100644 --- a/flake.nix +++ b/flake.nix @@ -243,9 +243,7 @@ # E2E test runner script - uses same fixed test environment as test-env (writeShellScriptBin "spago-test-e2e" '' set -euo pipefail - ${testEnv.envToExports testEnv.testEnv} - export PATH="${testEnv.testConfig.gitMock}/bin:$PATH" - export GIT_BINARY="${pkgs.git}/bin/git" + ${testEnv.testRuntimeExports} exec spago run -p registry-app-e2e '') ]; diff --git a/nix/test/config.nix b/nix/test/config.nix index e55d1746..e7975a40 100644 --- a/nix/test/config.nix +++ b/nix/test/config.nix @@ -94,6 +94,27 @@ let registryPkgs = pkgs.extend testOverlay; + # Centralized test runtime dependencies - use this in nativeBuildInputs + # to ensure all required binaries are available + testRuntimeInputs = registryPkgs.registry-runtime-deps ++ [ gitMock ]; + + # Centralized test runtime exports - use this in shell scripts to set up + # the complete test environment including PATH and GIT_BINARY. + # This is the single source of truth for test Git overrides. + testRuntimeExports = '' + ${envToExports testEnv} + export PATH="${lib.makeBinPath testRuntimeInputs}:$PATH" + export GIT_BINARY="${pkgs.git}/bin/git" + ''; + + # Complete build inputs for integration tests - combines runtime inputs with + # orchestration scripts. Use this in nativeBuildInputs for test derivations. + testBuildInputs = testRuntimeInputs ++ [ + wiremockStartScript + serverStartScript + setupGitFixtures + ]; + # Helper to create GitHub contents API response, as it returns base64-encoded content base64Response = { @@ -1003,8 +1024,8 @@ let serverStartScript = pkgs.writeShellScriptBin "start-server" '' set -e - # Set all test environment variables (from envDefaults + mock URLs). - ${envToExports testEnv} + # Set all test environment variables, PATH, and GIT_BINARY + ${testRuntimeExports} # STATE_DIR is required if [ -z "''${STATE_DIR:-}" ]; then @@ -1016,11 +1037,6 @@ let export DATABASE_URL="sqlite:$STATE_DIR/db/registry.sqlite3" export REPO_FIXTURES_DIR="$STATE_DIR/repo-fixtures" - # PATH setup for runtime deps and git mock - export PATH="${lib.makeBinPath registryPkgs.registry-runtime-deps}:$PATH" - export PATH="${gitMock}/bin:$PATH" - export GIT_BINARY="${pkgs.git}/bin/git" - mkdir -p "$STATE_DIR/db" # Always recreate git fixtures to ensure clean state @@ -1047,15 +1063,16 @@ in stateDir mockUrls testEnv - envToExports - gitMock testOverlay + testRuntimeInputs + testRuntimeExports + testBuildInputs wiremockConfigs combinedWiremockRoot - setupGitFixtures publishPayload wiremockStartScript serverStartScript + setupGitFixtures # For custom wiremock setups githubMappings storageMappings diff --git a/nix/test/integration.nix b/nix/test/integration.nix index 2a374348..7851a26f 100644 --- a/nix/test/integration.nix +++ b/nix/test/integration.nix @@ -39,13 +39,10 @@ else pkgs.nodejs pkgs.curl pkgs.jq - testSupport.gitMock pkgs.sqlite pkgs.nss_wrapper - testSupport.wiremockStartScript - testSupport.serverStartScript - testSupport.setupGitFixtures - ]; + ] + ++ testSupport.testBuildInputs; NODE_PATH = "${pkgs.registry-package-lock}/node_modules"; # Use nss_wrapper to resolve S3 bucket subdomain in the Nix sandbox. # The AWS SDK uses virtual-hosted style URLs (bucket.endpoint/key), so @@ -61,10 +58,9 @@ else export HOME=$TMPDIR export STATE_DIR=$TMPDIR/state export REPO_FIXTURES_DIR="$STATE_DIR/repo-fixtures" - export GIT_BINARY="${pkgs.git}/bin/git" - # Export test environment variables for E2E test runners - ${testSupport.envToExports testSupport.testEnv} + # Export test environment variables, PATH, and GIT_BINARY + ${testSupport.testRuntimeExports} mkdir -p $STATE_DIR diff --git a/nix/test/test-env.nix b/nix/test/test-env.nix index 8d854031..fbc757c3 100644 --- a/nix/test/test-env.nix +++ b/nix/test/test-env.nix @@ -89,8 +89,6 @@ let processComposeYaml = pkgs.writeText "process-compose.yaml" (builtins.toJSON processComposeConfig); - testEnvExports = testConfig.envToExports testConfig.testEnv; - # The state directory is fixed (not configurable) to avoid mismatch between # the test-env and spago-test-e2e shells. stateDir = testConfig.testEnv.STATE_DIR; @@ -102,8 +100,8 @@ let rm -rf ${stateDir} mkdir -p ${stateDir} - # Export all test environment variables - ${testEnvExports} + # Export all test environment variables, PATH, and GIT_BINARY + ${testConfig.testRuntimeExports} exec ${pkgs.process-compose}/bin/process-compose up \ -f ${processComposeYaml} \ @@ -122,14 +120,15 @@ in ; # Re-export commonly-used items from testConfig for convenience. - # This avoids verbose paths like `testEnv.testConfig.wiremockStartScript`. + # This avoids verbose paths like `testEnv.testConfig.testBuildInputs`. inherit (testConfig) + testEnv + testRuntimeInputs + testRuntimeExports + testBuildInputs wiremockStartScript serverStartScript setupGitFixtures - testEnv - envToExports - gitMock ; # Full testConfig still available for less common access patterns From 4dedf916b0e37e7c3f31d8ebbb1abc5f92ff77e8 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sun, 18 Jan 2026 13:45:19 -0500 Subject: [PATCH 19/20] address comments --- scripts/src/DailyImporter.purs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/scripts/src/DailyImporter.purs b/scripts/src/DailyImporter.purs index 5d07aad0..1cc19458 100644 --- a/scripts/src/DailyImporter.purs +++ b/scripts/src/DailyImporter.purs @@ -152,16 +152,9 @@ runDailyImport mode registryApiUrl = do let totalSubmitted = Array.foldl (+) 0 submitted Log.info $ "Daily Importer complete. Submitted " <> show totalSubmitted <> " publish jobs." --- | Submit a publish job for a new package version. --- | The compiler is not specified; the registry API will discover the oldest --- | compatible compiler based on the package's dependencies. -submitPublishJob - :: Mode - -> URL - -> PackageName - -> Version - -> String - -> Run DailyImportEffects Boolean +-- | Submit a publish job for a new package version. The compiler is not specified; the registry +-- | API will discover the latest compatible compiler based on the package's dependencies. +submitPublishJob :: Mode -> URL -> PackageName -> Version -> String -> Run DailyImportEffects Boolean submitPublishJob mode registryApiUrl name version ref = do let formatted = formatPackageVersion name version @@ -172,7 +165,7 @@ submitPublishJob mode registryApiUrl name version ref = do , version , location: Nothing -- Use current metadata location at publish time , ref - , compiler: Nothing -- Let the API discover the oldest compatible compiler + , compiler: Nothing -- Let the API discover the latest compatible compiler , resolutions: Nothing } From 677943ce18a90449b47997cc9a39e8d6a17176f0 Mon Sep 17 00:00:00 2001 From: Thomas Honeyman Date: Sun, 18 Jan 2026 14:08:25 -0500 Subject: [PATCH 20/20] fix new tarballs --- .../type-equality-4.0.2.tar.gz | Bin 2184 -> 1737 bytes .../unsafe-coerce-6.0.0.tar.gz | Bin 1310 -> 1094 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/app/fixtures/registry-storage/type-equality-4.0.2.tar.gz b/app/fixtures/registry-storage/type-equality-4.0.2.tar.gz index ba7126b6006c7f849f7252685f75bb22467248a9..75f4d4ec6dbce73811628bf5ca240ebd2ce7634d 100644 GIT binary patch literal 1737 zcmV;)1~&O0iwFP!000001MOICPvb}up3nXjB|gC@7)XFQvByfrX~0sjv$iuVx_!a1 zJJ36h+w1NSMR)Vxx2m0xm(32hBgCC{T@tumU0wCmOS?OzXC>d~zfO`|=-K|UeQ382 z-n}v%B7HbHF;Aq;=kLcyNA33EJ2-i5Alqo7bW)-Dx3KkpNP0{Dd!Ebuh`pK(O6uwH z@gL>CeR}d?BmeDo=fgXAlbrm`^Y6-XR*8p44ei6De?Wgy@hGiCsVytynNYeC!-*EM zfTYM^qBw{Gl$=y4H=Lm;l4=GcS&dZ-rb0i0tjtqRG>l~?Mj}l}LCZ=i4kfR~LTjEu zS;;4n;Xpqo8vk)rlILp%tO7^~p+E<75UyP8EXqCGa9cE~jEZAY-XH)KS58s8Bqi*a#;~ zE-;BBJ*7;~3&&C=PpNWnk)V32sAh_~dX%d1$Jbs2QE(aGIw1ob_ru_$*JWL}xP$nb zfgAMiLhtH2hU=i$Wnl!4-$kw;hu*~?4v?|qM7VN?CM?Il1NMDCWKje`2;NP<=bAnG&MqXGM#p)*c+*P82$*mnaI!b#{lH_jD{ ztlbwvf;n*qA-f^W7>8(Z5yf6Sh#6c3LDvL2V&O+TaiVXmUJ#jR4I_g_xQY=9!aGt*F)+bz!R<9een*zY@?az%wJz1!!NCK5 zuO~x0mE(iu;)5^i$?^Ovlu>gLb@w~_Zl2(=QBUIGDV4wzWrp(v4tb=5q|ulWLe`|p zzaEB7icrsIV^KWd;YRxZRjYpg{=j~}uw%BuSQ1s=F+!7uQ-@iX3TCOLoR;ki{# zo)K!U^qLm4p%$DnNk3sIYaX(GG*B}B)a4}}oxDf|Dt?Z@iYIvJDw&&j{B(-j_3&Bc zQl{Y3Ma+IWF*4+>^?A$ZW83~mP3jf5|BL&-(>dAP|3`->&HjH2THF6yHtQ3w=(-OGnMLi^0wM1GFMfue_iV*SB>1!fDcjgM+snP74cfN< zRh7PK59rVLf7-a9>HoY1{Zam7i9G0gd%W$o|7e;}sD50&5C6#nr_t9d&R`3mlnd(#iQAKuq% zs6}Lo)i&=7MI1(xBBeV=lEdmn;JKToMDdXGT3!^G@&b_!c`m0g6?snGyLrD(Pi2iN zsUA#|P1UQkgh?SrGxD^6dEX3w6aaehsgToR6_PJCKSSfQr7>fs;3-!ssYE^liC3#U zDNW0(UUiSxn*6?o)dD4-7puB2hnDJ4sYf}HTB&xJ`x4bs6&{GVTC5RvaQV| z!r2+Qnjn-Lqn4YrAH+TYHR_P5<&m+qHNtyl%Z#w`Sk4<_P}R fpN57SYN(-x8fvJah8k+=9pSugQjF7vBhGnz+tPd}#Tkw2uY!hSA-ILaR?hvtD< ztG?MW4iG0tNAef3`MXj+Y}D#luGJ4uYH#3Z*Me+AA@U$Y_Fqb8b8^2E}$Y$44g{_HP{34&K14DDe+&|0Yi#vS>WXp?*;N8hS;>yfBN>+%Q}= zj6|MAgCdW30zoo_LNJID5WL7jCTj*!5@ZiB;@MP`VHV{R;Fq4HAP36F=^A>qqTQteSLNNPooXTq2U4=`ZlAt*lwmJG3$ku#tm zQ=T(WTETHhFuKQXBW#eiKqHRw8CiPnIEF}BNS=d(BJ!ta?)7T*Bz73b%Zc8fK zv)%WYalCJgcGr{I>U(w>Irx@jj3iK1-ZxY}?|Y6^n$z)Z*X{Rwr`svv%s-%7P(}-f zwxo8u9XVChP}jX7qND?<{4#vFva#MJg-T6Xqyi5$*7R4qAwN{0zcNSY*q3eR((W{E zYV8u04~}P-&{mE|9grlk{a_)_zML}I2&Ga*$1@flTb7mt=K`$OdxylHcShTL&fHI= z%9>Ymb(_Y{*Z*z#-?gpQwQWv^JL2(7{;xGojyLoF32q5={(l9$g%nf1kSV{i<_w1M z@lPMym`-F$i&-Gp5BrlmPfx4WXgW5c)>`NNW}9Ziidz!%nR{iiZhg5j8vgx?LaAPFaD(?DQW zeYw0rD0BOFKNl=}I9u@gbHi#ne^D&3JZC~v!d;jsqI}6ORTbuvgUPdTH4VbMV9dnR zoaIzXUrwlm=@!3X@lClOHxszg%7tOPeG3kL25~HRB!+Q&dy6SN7<1gQ$a+|JEo!Gx zt<{Z;VO-+|A>Q2H;P&8SS`2WjG{HRsR;+2~OXl<&ET_LaY+L^ae1`koe+ZuJhQ~AO z|G~-8=K6nlRIBUt|26PW17MGK^?RqV_f(|pm8sW+PZ`Fv?D?{VP|=C;{F4Yt=G<#l z8kJ@o;ITn9;^B#l2u~Cb*iOP>9nqlP$V_&^sv?VDE`~{jkj^iXV>HIY4blIoNWXhm zHQz1x$WZtR>83ov6-g}W6FhGIG?mNMV;!Db(&AA<%A0FcWI!!y=8Kc6!>S*oC_{GN zVksV-SQ17^_)!BZGogi-$5J0Z9%Hy3Jd#}YN!V_{X5%jQ_?_O~?OhU@iY| zN!Ir)qw6+mmCo@iQh3bOh3_&xReJ8u5|*uRGZulUI=GnoKe-KASqavcfj?N#?hxDJ zUu5A<-vHGf|8w5|8g>$5ZyF)BdiU;PdT&SU=G3zpsK%+TUIrm}p1vavsm* z|Jre*v0?w?liH!q|E~dch6OUWxiB|o*nJ4&KnPhcs|8k;=?RaIWGUzJho11TP=eWn zWy~-lJSb;5^l)cwHaVl0*?mPxeno?rl?(&V0RIA?9{$Mv=B)5P%W^S)4#s)`x52G^ z54|1SsxkCLD4o@m-wQ$vMnxj;Sc4c=_XdxzsUk|oF;o5Jm1KgF4Oq-)FpJ`tUT^2` zwfT%Ise)`QZy|*mm2)VPX!JlcOr%He;jsR*6*x!5;?;^)8E?l&Zvp7x7RHN_bB1u0uA85;mk4Vmwu4 z$bt#e*fb)Xp3+q5LiWytJE(lC{xlzNbBzsaK>!2z>)9#|szFoGxIr1q_#4da56VMU z*rX<;sxKY@(LWaIps7-~HM*N*0yu-fFy_}8;B(3Qj0*dTKROc{X2}lQ%}rU)zMTt8 zUC$s41e?>}CqjRhjX|kIO2eU6q~FX)w8@s-7WHDlx~->;wLt-7nu>)aK&HU6?{ zF2R3$Y0%Kn(9qD((9qD((9qD((9qD((9qD((9qD((9qD((9qD((9qD((9rlF#J>Rp KEZV;SPyhfn&`zfS diff --git a/app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz b/app/fixtures/registry-storage/unsafe-coerce-6.0.0.tar.gz index 34451628ffbbc56bec9e79b41277f5b85877497d..1b91c1a65ceabaaaecf5d271a75b9fe55718b1ab 100644 GIT binary patch literal 1094 zcmV-M1iAYkiwFP!000001MOICZyPrZ&S(D$B46?X`)tQ{3Z&?Orp18m%Z6sdJ`7td z>v%qOchW)9`DJVV`zXnFe6HJUnPIG`9w4zp@kol~a}rZa!!qn}g_>jMI_V|7?#Y4C zBfJ=m{3l`e^W*RRe*bE40;59*vJbVfOq2U#`1&8B-c$dHnxjtM8zm3p!xi;ObfYh9_{8ntbK;|EmRXDp`rAH*i|j8V%Q?vYkd>oVJ}-u1c)f)NAI}R_3PL zoeG=RlZ30X`}lD8^L7fsQ5tXN_U>(G(0Nm^+N>5eR>}yg^pe(Ha3r(K%4?G|uSB9+ z7taqPafZmwgZC(=QfX)Z*I@ecvYTA4{dg+elif;5l-8`}Ql)rbsP>_e zjcr)*gj!1~#{ZyH1{sNd^P!|Tdy5sy6eSmAc$0yxop2?MDttbE_jL6;X+5-_SO0zd zCvWC}^na57gTZK*|Gj=M^8XlU@P7|kXQ+*!9;MaLZBJ9UP@LKD%&L6@J%5->)Xmq- znhrtX9dZA&ub~ZRHD*q6wiQykL!9SR;;X_w{u|8?dIx;Q{m)$q;{ERk_*DIW_VUBD z!N0FR-T&hK|0sB({%^vC>Am?cKAt-N-LTiJ|M2>H6wm)-pxcFTjfGJ0BvicB!bPRD z1$W)F5V|BR!7IpW$=wML=Zgz}#%r{-maur)dUzR6-7g9MjQ?&Tc)pJ(_D3+OH((?$=weAef+dX= zJ%rNw!3Hv=VL~zah|G4tiPsMg-6D5GQ8P-Xj0>xo(y5S43(8?u;6h9aY_Ik&Gs?82 zw0koylT(e%lBTIMT1((NztdF-G;&pzZus%3PB9abqN$JzR){}+$Ocpv%r&FYEMzuk z>zq;&ndu0Vz_+wZT(Gh#@MddU&Rmv#pPL%~ybJwQa{g=@Q68c5;o5a&RA|fIpA|fIpA|fIpBI3)( M-&?o{^#CXU0DBKJGynhq literal 1310 zcmV+(1>yQ1iwFP!000001MOOEZ`(Ey&S(D$B43gQu`Kx&NRa_ei(%`R70ZTw7`B*} zP8K_f6iCW1+w$Lcq-;laoqBd$ZH4&&jvexJcf8~Ap-FqK4|Mh&&*0F7vq8{{m&)5Iv=O75$oioxqwjDWppTIcGf)fzLM6eBMbS?lTH*5~!4ECjBsn!EdV%}(=e zN(;5w2*V`H_l<3p66B?jaeaVDr9=N8=g^wdLFGsHs(Sa{lAu~udZ6&)zXeF zg$LA|3l3bXWjhxUJY*s&QdqT#Lg&TGLk&u=X|V;hY{_Z(2Zpk2!=~S?D-lfIK@L0u z9x_zCia=!~q2NlSHXgs*a6`#-n-e|fm||FrR3{g*|592(ev{qF=F zcmG!lcI3Z818>W-(~oLz8~d;S1K-^Ly7m9d!>#|PjUDxWc+pp|tNzW8J=W{L+wb?> z`ClDqwMco5MaqnOWyTu~nB_uhVy>GsDQ8L9MQ%x4@X(y_k;~aDd&Vb07m}0N>&D}2 zw}yO8*qi^&<-ndc4&eW;@4o-l0>}TSjep1g2NZn;JL|7MG)Ocx> zW7Dx`8xI8t6ZM;oOk2{}FlT-%FH5(`O2 z=#%wGRUP)MeEHBFgyvFo-Af}3GcBo*5#v-cjA5L@jEz#bUW}i|7-^1Sw`xoz#}cSU zNmazy=aKL2NEa!P5Q;1_m!CG&5yXt6H#vlirtDAavLYfb)vbFYX&KpsE@O&8q^il3 zN4~{T!i;8l3OAKu4Tr<03)-?NmLj_oA_dfPb3>LNo8OT4O7Kr*Zr5kmQf-ppZ4)@*;7h{bBP?#wtaCVBgR5dlzG&H{wOPNa0`h?NN zsHy`U+FWumHI=a`Pln+_$V6)Kbmbau18ynzE6ka(;nP5Mhz6P0>uqYmU0828QqyPW_