[Infrastructure] Fix NPM packaging (#54056)

* Marks the workspace projects as shipping
* Moves the cache setup location to the build via environment variables
* Cleans up the packaging script.
* Updates the build to stamp the version on SignalR bundles.

.azure/pipelines/components-e2e-tests.yml

@@ -25,6 +25,8 @@ variables:
   value: true
 - name: _TeamName
   value:  AspNetCore
+- name: npm_config_cache
+  value: src/submodules/Node-Externals/cache
 - template: /eng/common/templates/variables/pool-providers.yml
 
 jobs:
@@ -41,7 +43,7 @@ jobs:
       displayName: Update submodules
     - script: ./restore.sh
       displayName: Run restore.sh
-    - script: npm ci  --offline
+    - script: npm ci --offline
       displayName: NPM install
     - script: npm run build
       displayName: Build JS

.npmrc

@@ -1,3 +1,2 @@
 registry=https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public-npm/npm/registry/
-cache=./src/submodules/Node-Externals/cache
 always-auth=true

eng/Npm.Workspace.nodeproj

@@ -4,6 +4,7 @@
   <Import Project="$(MSBuildThisFileDirectory)..\Directory.Build.props" />
 
   <PropertyGroup>
+    <IsShippingPackage>true</IsShippingPackage>
     <ExcludeFromSourceBuild>false</ExcludeFromSourceBuild>
     <IsTestProject>true</IsTestProject>
     <IsUnitTestProject>true</IsUnitTestProject>
@@ -12,22 +13,42 @@
     <TestDependsOnAspNetRuntime>false</TestDependsOnAspNetRuntime>
     <ContainsFunctionalTestAssets>false</ContainsFunctionalTestAssets>
     <BuildHelixPayload>false</BuildHelixPayload>
-    <PackageVersion>$(VersionPrefix)$(VersionSuffix)</PackageVersion>
+    <_NpmAdditionalEnvironmentVariables>npm_config_cache=$(RepoRoot)src/submodules/Node-Externals/cache</_NpmAdditionalEnvironmentVariables>
   </PropertyGroup>
 
-  <Target Name="_VerifyNPMCache">
+  <Target Name="_VerifyNPMCache" Condition="$(ContinuousIntegrationBuild) == 'true'" >
     <Message Text="Verifying NPM cache..." Importance="high" />
-    <Exec Command="npm cache verify" WorkingDirectory="$(MSBuildThisFileDirectory).." />
+    <Exec
+      Command="npm cache verify"
+      WorkingDirectory="$(MSBuildThisFileDirectory).."
+      EnvironmentVariables="$(_NpmAdditionalEnvironmentVariables)" />
   </Target>
 
   <Target Name="Restore" DependsOnTargets="_VerifyNPMCache">
     <Message Text="Restoring NPM packages..." Importance="high" />
-    <Exec Command="npm ci --offline" WorkingDirectory="$(MSBuildThisFileDirectory).." />
+    <Exec
+      Command="npm ci --offline"
+      WorkingDirectory="$(MSBuildThisFileDirectory).."
+      EnvironmentVariables="$(_NpmAdditionalEnvironmentVariables)" />
   </Target>
 
   <Target Name="Build">
+    <PropertyGroup>
+      <PackageVersion>$(VersionPrefix)$(VersionSuffix)</PackageVersion>
+    </PropertyGroup>
+    <MakeDir Directories="$(IntermediateOutputPath)" Condition="!Exists('$(IntermediateOutputPath)')" />
+
     <Message Text="Building NPM packages..." Importance="high" />
-    <Exec Command="npm run build" WorkingDirectory="$(MSBuildThisFileDirectory).." />
+
+    <Exec
+      Condition="$(ContinuousIntegrationBuild) == 'true'"
+      Command="node $(MSBuildThisFileDirectory)scripts/npm/pack-workspace.mjs --update-versions $(RepoRoot)package.json $(PackageVersion) $(PackageOutputPath) $(IntermediateOutputPath)"
+      EnvironmentVariables="$(_NpmAdditionalEnvironmentVariables)" />
+
+    <Exec
+      Command="npm run build"
+      WorkingDirectory="$(MSBuildThisFileDirectory).."
+      />
   </Target>
 
   <Target Name="_Test" BeforeTargets="Test">
@@ -36,9 +57,19 @@
   </Target>
 
   <Target Name="Pack">
+    <PropertyGroup>
+      <PackageVersion>$(VersionPrefix)$(VersionSuffix)</PackageVersion>
+    </PropertyGroup>
     <Message Text="Packing NPM packages..." Importance="high" />
     <MakeDir Directories="$(PackageOutputPath)" Condition="!Exists('$(PackageOutputPath)')" />
-    <Exec Command="node $(MSBuildThisFileDirectory)scripts\pack-workspace.js $(RepoRoot)package.json $(PackageVersion) $(PackageOutputPath)" />
+    <Exec Command="node $(MSBuildThisFileDirectory)scripts/npm/pack-workspace.mjs --create-packages $(RepoRoot)package.json $(PackageVersion) $(PackageOutputPath) $(IntermediateOutputPath)" />
+    <ItemGroup>
+      <_NpmGeneratedPackages Include="$(PackageOutputPath)/*.tgz" />
+    </ItemGroup>
+    <PropertyGroup>
+      <_NpmFiles>@(_NpmGeneratedPackages->'📦 $(PackageOutputPath)%(FileName)%(Extension)', '%0a  ')</_NpmFiles>
+    </PropertyGroup>
+    <Message Importance="High" Text="$(MSBuildProjectName) -> %0a%20%20$(_NpmFiles)" />
   </Target>
 
   <!-- Import Directory.Build.targets -->

eng/SourceBuild.props

@@ -109,7 +109,7 @@
 
     <PropertyGroup>
       <!-- Disable installing puppeteer browsers when running in source build -->
-      <_AdditionalEnvironmentVariable Condition="$(ArcadeBuildFromSource) == 'true'">PUPPETEER_SKIP_DOWNLOAD=1</_AdditionalEnvironmentVariable>
+      <_AdditionalEnvironmentVariable Condition="$(ArcadeBuildFromSource) == 'true'">PUPPETEER_SKIP_DOWNLOAD=1;npm_config_cache=$(RepoRoot)src/submodules/Node-Externals/cache</_AdditionalEnvironmentVariable>
     </PropertyGroup>
 
     <Exec

eng/scripts/npm/pack-workspace.mjs

@@ -0,0 +1,85 @@
+// Handles packing of the packages in the workspace
+// The script iterates over all public packages in the workspace, updates their versions, and then packs them into tarballs.
+import { resolve, dirname } from 'path';
+import { execSync } from 'child_process';
+import fsExtra from 'fs-extra';
+const { existsSync, writeJsonSync, readJsonSync, moveSync } = fsExtra;
+import { applyVersions } from './update-dependency-versions.mjs';
+
+// Valid actions are --update-versions and --create-packages
+const action = process.argv[2];
+
+const workspacePath = process.argv[3];
+
+const defaultPackageVersion = process.argv[4];
+
+const packageOutputPath = process.argv[5];
+
+const intermediateOutputPath = process.argv[6];
+
+// Validate and throw if the arguments are not provided
+if (!workspacePath) {
+  throw new Error('The workspace path was not provided.');
+}
+
+if (!defaultPackageVersion) {
+  throw new Error('The default package version was not provided.');
+}
+
+if (!packageOutputPath) {
+  throw new Error('The package output path was not provided.');
+}
+
+if (!intermediateOutputPath) {
+  throw new Error('The intermediate output path was not provided.');
+}
+
+// Log all the captured process arguments
+console.log(`Workspace Path: ${workspacePath}`);
+console.log(`Default Package Version: ${defaultPackageVersion}`);
+console.log(`Package Output Path: ${packageOutputPath}`);
+console.log(`Intermediate Output Path: ${intermediateOutputPath}`);
+
+if (action === '--update-versions') {
+  const [packagesToPack, renames] = applyVersions(defaultPackageVersion, workspacePath);
+  // Write packagesToPack and renames to a file
+  writeJsonSync(resolve(intermediateOutputPath, 'packagesToPack.json'), packagesToPack);
+  writeJsonSync(resolve(intermediateOutputPath, 'renames.json'), renames);
+} else if (action === '--create-packages') {
+  if (!existsSync(packageOutputPath)) {
+    throw new Error(`The package output path ${packageOutputPath} does not exist.`);
+  }
+
+  // Read packagesToPack and renames from a file
+  const packagesToPack = readJsonSync(resolve(intermediateOutputPath, 'packagesToPack.json'));
+  const renames = readJsonSync(resolve(intermediateOutputPath, 'renames.json'));
+  try {
+    createPackages(packagesToPack);
+  } finally {
+    // Restore the package.json files to their "originals"
+    for (const [from, to] of renames) {
+      moveSync(from, to, { overwrite: true });
+    }
+  }
+} else {
+  throw new Error(`The action ${action} is not supported.`);
+}
+
+// For each package to pack run npm pack
+function createPackages(packagesToPack) {
+  for (const [packagePath, packageJson] of packagesToPack) {
+    const packageName = packageJson.name;
+    const packageVersion = defaultPackageVersion;
+    const packageDir = dirname(packagePath);
+    const normalizedPackageName = packageName.replace('@', '').replace('/', '-');
+    const packageFileName = `${normalizedPackageName}-${packageVersion}.tgz`;
+    const packageTarball = resolve(packageDir, `${packageFileName}`);
+    console.log(`Packing ${packageName}...`);
+    // Log and execute the command
+    console.log(`npm pack ${packageDir} --pack-destination ${packageOutputPath}`);
+    execSync(`npm pack ${packageDir} --pack-destination ${packageOutputPath}`, { stdio: 'inherit' });
+
+    console.log(`Packed ${packageName} to ${resolve(packageOutputPath, packageTarball)}`);
+  }
+}
+

eng/scripts/pack-workspace.js → eng/scripts/npm/update-dependency-versions.mjs

@@ -1,73 +1,46 @@
-const path = require('path');
-const { execSync } = require('child_process');
-const fs = require('fs-extra');
+import path from 'path';
+import { execSync } from 'child_process';
+import fs from 'fs-extra';
 
-// Get the path to the workspace package.json from the process arguments list
-const workspacePath = process.argv[2];
-const workspacePackage = require(workspacePath);
+export function applyVersions(defaultPackageVersion, workspacePath) {
+  // Get the workspace package.json from the provided path
+  const workspacePackage = fs.readJsonSync(workspacePath);
 
-// Get the package version from the process arguments list
-const defaultPackageVersion = process.argv[3];
+  // Get the workspace directory
+  const workspaceDir = path.dirname(workspacePath);
 
-// Get the package output path from the process arguments list
-const packageOutputPath = process.argv[4];
-
-// Get the workspace directory
-const workspaceDir = path.dirname(workspacePath);
-
-// Validate and throw if the arguments are not provided
-if (!workspacePath) {
-  throw new Error('The workspace path was not provided.');
-}
-
-if (!defaultPackageVersion) {
-  throw new Error('The default package version was not provided.');
-}
-
-if (!packageOutputPath) {
-  throw new Error('The package output path was not provided.');
-}
-
-// Log all the captured process arguments
-console.log(`Workspace Path: ${workspacePath}`);
-console.log(`Default Package Version: ${defaultPackageVersion}`);
-console.log(`Package Output Path: ${packageOutputPath}`);
-
-if (!fs.existsSync(packageOutputPath)) {
-  throw new Error(`The package output path ${packageOutputPath} does not exist.`);
-}
+  // Validate and throw if the arguments are not provided
+  if (!workspacePath) {
+    throw new Error('The workspace path was not provided.');
+  }
 
-const packages = workspacePackage.workspaces;
-const packagesToPack = [];
-for (const package of packages) {
-  const packagePath = path.resolve(workspaceDir, package, 'package.json');
-  const packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf-8'));
-  const { execSync } = require('child_process');
-  if (!packageJson.private) {
-    packagesToPack.push([packagePath, packageJson]);
-  } else {
-    console.log(`Skipping ${packageJson.name} because it is marked as private.`);
+  if (!defaultPackageVersion) {
+    throw new Error('The default package version was not provided.');
   }
-}
 
-const currentDir = process.cwd();
-const renames = [];
+  const packages = workspacePackage.workspaces;
+  const packagesToPack = [];
+  for (const pkg of packages) {
+    const packagePath = path.resolve(workspaceDir, pkg, 'package.json');
+    const packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf-8'));
+    if (!packageJson.private) {
+      packagesToPack.push([packagePath, packageJson]);
+    } else {
+      console.log(`Skipping ${packageJson.name} because it is marked as private.`);
+    }
+  }
 
-try {
   // For each package to be packed, run npm version to apply the version
-  applyPackageVersion(packagesToPack);
+  var renames = applyPackageVersion(packagesToPack, defaultPackageVersion);
 
   updateDependencyVersions(packagesToPack);
 
-  createPackages(packagesToPack);
-} finally {
-  // Restore the package.json files to their "originals"
-  for (const [from, to] of renames) {
-    fs.moveSync(from, to, { overwrite: true });
-  }
+  return [packagesToPack, renames];
 }
 
-function applyPackageVersion(packagesToPack) {
+function applyPackageVersion(packagesToPack, defaultPackageVersion) {
+  const currentDir = process.cwd();
+  const renames = [];
   for (const [packagePath, packageJson] of packagesToPack) {
     const packageName = packageJson.name;
     const packageVersion = defaultPackageVersion;
@@ -83,24 +56,8 @@ function applyPackageVersion(packagesToPack) {
     process.chdir(currentDir);
     console.log(`Applied version ${packageVersion} to ${packageName} in ${packageDir}...`);
   }
-}
 
-// For each package to pack run npm pack
-function createPackages(packagesToPack) {
-  for (const [packagePath, packageJson] of packagesToPack) {
-    const packageName = packageJson.name;
-    const packageVersion = defaultPackageVersion;
-    const packageDir = path.dirname(packagePath);
-    const normalizedPackageName = packageName.replace('@', '').replace('/', '-');
-    const packageFileName = `${normalizedPackageName}-${packageVersion}.tgz`;
-    const packageTarball = path.resolve(packageDir, `${packageFileName}`);
-    console.log(`Packing ${packageName}...`);
-    // Log and execute the command
-    console.log(`npm pack ${packageDir} --pack-destination ${packageOutputPath}`);
-    execSync(`npm pack ${packageDir} --pack-destination ${packageOutputPath}`, { stdio: 'inherit' });
-
-    console.log(`Packed ${packageName} to ${path.resolve(packageOutputPath, packageTarball)}`);
-  }
+  return renames;
 }
 
 // For each package to be packed, update the version for the dependencies that are part of the workspace.
@@ -139,4 +96,3 @@ function updateDependencyVersions(packagesToPack) {
     }
   }
 }
-

src/SignalR/clients/ts/.gitignore

@@ -0,0 +1 @@
+**/pkg-version.ts

src/SignalR/clients/ts/signalr-protocol-msgpack/package.json

@@ -14,6 +14,7 @@
   "sideEffects": false,
   "scripts": {
     "clean": "rimraf ./dist",
+    "prebuild": "rimraf ./src/pkg-version.ts && node -e \"const fs = require('fs'); const packageJson = require('./package.json'); fs.writeFileSync('./src/pkg-version.ts', 'export const VERSION = \\'' + packageJson.version + '\\';');\"",
     "build": "npm run build:esm && npm run build:cjs && npm run build:browser && npm run build:uglify",
     "build:esm": "tsc --project ./tsconfig.json --module es2015 --outDir ./dist/esm -d",
     "build:cjs": "tsc --project ./tsconfig.json --module commonjs --outDir ./dist/cjs",

src/SignalR/clients/ts/signalr-protocol-msgpack/src/index.ts

@@ -3,7 +3,7 @@
 
 // Version token that will be replaced by the prepack command
 /** The version of the SignalR Message Pack protocol library. */
-export const VERSION = "0.0.0-DEV_BUILD";
+export { VERSION } from "./pkg-version";
 
 export { MessagePackHubProtocol } from "./MessagePackHubProtocol";
 

src/SignalR/clients/ts/signalr/package.json

@@ -14,6 +14,7 @@
   "sideEffects": false,
   "scripts": {
     "clean": "rimraf ./dist",
+    "prebuild": "rimraf ./src/pkg-version.ts && node -e \"const fs = require('fs'); const packageJson = require('./package.json'); fs.writeFileSync('./src/pkg-version.ts', 'export const VERSION = \\'' + packageJson.version + '\\';');\"",
     "build": "npm run build:esm && npm run build:cjs && npm run build:browser && npm run build:webworker",
     "build:esm": "tsc --project ./tsconfig.json --module es2015 --outDir ./dist/esm -d",
     "build:cjs": "tsc --project ./tsconfig.json --module commonjs --outDir ./dist/cjs",

src/SignalR/clients/ts/signalr/src/Utils.ts

@@ -7,11 +7,12 @@ import { NullLogger } from "./Loggers";
 import { IStreamSubscriber, ISubscription } from "./Stream";
 import { Subject } from "./Subject";
 import { IHttpConnectionOptions } from "./IHttpConnectionOptions";
+import { VERSION } from "./pkg-version";
 
 // Version token that will be replaced by the prepack command
 /** The version of the SignalR client. */
 
-export const VERSION: string = "0.0.0-DEV_BUILD";
+export { VERSION };
 /** @private */
 export class Arg {
     public static isRequired(val: any, name: string): void {