Skip to content

Commit 85355c1

Browse files
committed
Add dts bundling
1 parent 798841e commit 85355c1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+7820
-7783
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ scripts/configurePrerelease.js
4747
scripts/configureLanguageServiceBuild.js
4848
scripts/open-user-pr.js
4949
scripts/open-cherry-pick-pr.js
50+
scripts/dtsBundler.js
5051
scripts/processDiagnosticMessages.d.ts
5152
scripts/processDiagnosticMessages.js
5253
scripts/produceLKG.js

Gulpfile.js

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,20 @@ const buildSrc = () => buildProject("src");
126126

127127
task("build-src", series(preSrc, buildSrc));
128128

129+
/**
130+
* @param {string} entrypoint
131+
* @param {string} output
132+
*/
133+
async function runDtsBundler(entrypoint, output) {
134+
await exec(process.execPath, [
135+
"./scripts/dtsBundler.js",
136+
"--entrypoint",
137+
entrypoint,
138+
"--output",
139+
output,
140+
]);
141+
}
142+
129143
/** @type {string | undefined} */
130144
let copyrightHeader;
131145
function getCopyrightHeader() {
@@ -268,6 +282,7 @@ const esbuildServices = esbuildTask("./src/typescript/typescript.ts", "./built/l
268282
const writeServicesCJSShim = () => writeCJSReexport("./built/local/typescript/typescript.js", "./built/local/typescript.js");
269283
const buildServicesProject = () => buildProject("src/typescript");
270284

285+
// TODO(jakebailey): rename this; no longer "services".
271286
const buildServices = () => {
272287
if (cmdLineOptions.bundle) return esbuildServices.build();
273288
writeServicesCJSShim();
@@ -297,6 +312,9 @@ task("watch-services").flags = {
297312
" --built": "Compile using the built version of the compiler."
298313
};
299314

315+
const dtsServices = () => runDtsBundler("./built/local/typescript/typescript.d.ts", "./built/local/typescript.d.ts");
316+
task("dts-services", series(preBuild, buildServicesProject, dtsServices));
317+
task("dts-services").description = "Builds typescript.d.ts";
300318

301319
const esbuildServer = esbuildTask("./src/tsserver/server.ts", "./built/local/tsserver.js", /* exportIsTsObject */ true);
302320
const writeServerCJSShim = () => writeCJSReexport("./built/local/tsserver/server.js", "./built/local/tsserver.js");
@@ -348,10 +366,11 @@ task("watch-min").flags = {
348366
const esbuildLssl = esbuildTask("./src/tsserverlibrary/tsserverlibrary.ts", "./built/local/tsserverlibrary.js", /* exportIsTsObject */ true);
349367
const writeLsslCJSShim = () => writeCJSReexport("./built/local/tsserverlibrary/tsserverlibrary.js", "./built/local/tsserverlibrary.js");
350368

369+
const buildLsslProject = () => buildProject("src/tsserverlibrary");
351370
const buildLssl = () => {
352371
if (cmdLineOptions.bundle) return esbuildLssl.build();
353372
writeLsslCJSShim();
354-
return buildProject("src/tsserverlibrary");
373+
return buildLsslProject();
355374
};
356375
task("lssl", series(preBuild, buildLssl));
357376
task("lssl").description = "Builds language service server library";
@@ -375,6 +394,14 @@ task("watch-lssl").flags = {
375394
" --built": "Compile using the built version of the compiler."
376395
};
377396

397+
const dtsLssl = () => runDtsBundler("./built/local/tsserverlibrary/tsserverlibrary.d.ts", "./built/local/tsserverlibrary.d.ts");
398+
task("dts-lssl", series(preBuild, buildLsslProject, dtsLssl));
399+
task("dts-lssl").description = "Builds tsserverlibrary.d.ts";
400+
401+
// TODO(jakebailey): this is probably not efficient, but, gulp.
402+
const dts = series(preBuild, parallel(buildServicesProject, buildLsslProject), parallel(dtsServices, dtsLssl));
403+
task("dts", dts);
404+
378405
const testRunner = "./built/local/run.js";
379406
const esbuildTests = esbuildTask("./src/testRunner/_namespaces/Harness.ts", testRunner);
380407
const writeTestsCJSShim = () => writeCJSReexport("./built/local/testRunner/runner.js", testRunner);
@@ -488,7 +515,7 @@ const buildOtherOutputs = parallel(buildCancellationToken, buildTypingsInstaller
488515
task("other-outputs", series(preBuild, buildOtherOutputs));
489516
task("other-outputs").description = "Builds miscelaneous scripts and documents distributed with the LKG";
490517

491-
task("local", series(preBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs)));
518+
task("local", series(preBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs, dts)));
492519
task("local").description = "Builds the full compiler and services";
493520
task("local").flags = {
494521
" --built": "Compile using the built version of the compiler."
@@ -504,7 +531,7 @@ const preTest = parallel(buildTsc, buildTests, buildServices, buildLssl);
504531
preTest.displayName = "preTest";
505532

506533
const runTests = () => runConsoleTests(testRunner, "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
507-
task("runtests", series(preBuild, preTest, runTests));
534+
task("runtests", series(preBuild, preTest, dts, runTests));
508535
task("runtests").description = "Runs the tests using the built run.js file.";
509536
task("runtests").flags = {
510537
"-t --tests=<regex>": "Pattern for tests to run.",
@@ -523,7 +550,7 @@ task("runtests").flags = {
523550
};
524551

525552
const runTestsParallel = () => runConsoleTests(testRunner, "min", /*runInParallel*/ cmdLineOptions.workers > 1, /*watchMode*/ false);
526-
task("runtests-parallel", series(preBuild, preTest, runTestsParallel));
553+
task("runtests-parallel", series(preBuild, preTest, dts, runTestsParallel));
527554
task("runtests-parallel").description = "Runs all the tests in parallel using the built run.js file.";
528555
task("runtests-parallel").flags = {
529556
" --light": "Run tests in light mode (fewer verifications, but tests run faster).",
@@ -613,8 +640,7 @@ const produceLKG = async () => {
613640
}
614641
};
615642

616-
// TODO(jakebailey): dependencies on dts
617-
task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs), produceLKG));
643+
task("LKG", series(lkgPreBuild, parallel(localize, buildTsc, buildServer, buildServices, buildLssl, buildOtherOutputs, dts), produceLKG));
618644
task("LKG").description = "Makes a new LKG out of the built js files";
619645
task("LKG").flags = {
620646
" --built": "Compile using the built version of the compiler.",

scripts/build/options.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ module.exports = minimist(process.argv.slice(2), {
4141
dirty: false,
4242
built: false,
4343
ci,
44-
bundle: true
44+
bundle: true,
4545
}
4646
});
4747

scripts/dtsBundler.ts

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
/**
2+
* WARNING: this is a very, very rudimentary d.ts bundler; it only works
3+
* in the TS project thanks to our history using namespaces, which has
4+
* prevented us from duplicating names across files, and allows us to
5+
* bundle as namespaces again, even though the project is modules.
6+
*/
7+
8+
import * as assert from "assert";
9+
import * as fs from "fs";
10+
import * as path from "path";
11+
import * as minimist from "minimist";
12+
import * as ts from "../lib/typescript";
13+
14+
const dotDts = ".d.ts";
15+
16+
const options = minimist(process.argv.slice(2), {
17+
string: ["project", "entrypoint", "output"],
18+
});
19+
20+
const entrypoint = options.entrypoint;
21+
const output = options.output;
22+
23+
assert(typeof entrypoint === "string" && entrypoint);
24+
assert(typeof output === "string" && output);
25+
assert(output.endsWith(dotDts));
26+
27+
const internalOutput = output.substring(0, output.length - dotDts.length) + ".internal" + dotDts;
28+
29+
console.log(`Bundling ${entrypoint} to ${output} and ${internalOutput}`);
30+
31+
const newLineKind = ts.NewLineKind.LineFeed;
32+
const newLine = newLineKind === ts.NewLineKind.LineFeed ? "\n" : "\r\n";
33+
34+
function isDeclarationStatement(node: ts.Node): node is ts.DeclarationStatement {
35+
return (ts as any).isDeclarationStatement(node);
36+
}
37+
38+
function isInternalDeclaration(node: ts.Node): boolean {
39+
return (ts as any).isInternalDeclaration(node, node.getSourceFile());
40+
}
41+
42+
function getParentVariableStatement(node: ts.VariableDeclaration): ts.VariableStatement {
43+
const declarationList = node.parent as ts.VariableDeclarationList;
44+
assert(ts.isVariableDeclarationList(declarationList), `expected VariableDeclarationList at ${nodeToLocation(node)}`);
45+
assert(declarationList.declarations.length === 1, `expected VariableDeclarationList of length 1 at ${nodeToLocation(node)}`);
46+
const variableStatement = declarationList.parent;
47+
assert(ts.isVariableStatement(variableStatement), `expected VariableStatement at ${nodeToLocation(node)}`);
48+
return variableStatement;
49+
}
50+
51+
function getDeclarationStatement(node: ts.Declaration): ts.Statement | undefined {
52+
if (ts.isVariableDeclaration(node)) {
53+
return getParentVariableStatement(node);
54+
}
55+
else if (isDeclarationStatement(node)) {
56+
return node;
57+
}
58+
return undefined;
59+
}
60+
61+
const nullTransformationContext: ts.TransformationContext = (ts as any).nullTransformationContext;
62+
63+
const program = ts.createProgram([entrypoint], { target: ts.ScriptTarget.ES5 });
64+
65+
const typeChecker = program.getTypeChecker();
66+
67+
const sourceFile = program.getSourceFile(entrypoint);
68+
assert(sourceFile);
69+
const moduleSymbol = typeChecker.getSymbolAtLocation(sourceFile);
70+
assert(moduleSymbol);
71+
72+
const printer = ts.createPrinter({ newLine: newLineKind });
73+
74+
75+
const publicLines: string[] = [];
76+
const internalLines: string[] = [];
77+
78+
const indent = " ";
79+
let currentIndent = "";
80+
81+
function increaseIndent() {
82+
currentIndent += indent;
83+
}
84+
85+
function decreaseIndent() {
86+
currentIndent = currentIndent.slice(indent.length);
87+
}
88+
89+
enum WriteTarget {
90+
Public = 1 << 0,
91+
Internal = 1 << 1,
92+
Both = Public | Internal,
93+
}
94+
95+
function write(s: string, target: WriteTarget) {
96+
if (!target) {
97+
return;
98+
}
99+
100+
const toPush = !s ? [""] : s.split(/\r?\n/).filter(line => line).map(line => (currentIndent + line).trimEnd());
101+
102+
if (target & WriteTarget.Public) {
103+
publicLines.push(...toPush);
104+
}
105+
if (target & WriteTarget.Internal) {
106+
internalLines.push(...toPush);
107+
}
108+
}
109+
110+
function writeNode(node: ts.Node, sourceFile: ts.SourceFile, target: WriteTarget) {
111+
write(printer.printNode(ts.EmitHint.Unspecified, node, sourceFile), target);
112+
}
113+
114+
const containsPublicAPICache = new Map<ts.Symbol, boolean>();
115+
116+
function containsPublicAPI(symbol: ts.Symbol): boolean {
117+
const cached = containsPublicAPICache.get(symbol);
118+
if (cached !== undefined) {
119+
return cached;
120+
}
121+
122+
const result = containsPublicAPIWorker();
123+
containsPublicAPICache.set(symbol, result);
124+
return result;
125+
126+
function containsPublicAPIWorker(): boolean {
127+
if (!symbol.declarations?.length) {
128+
return false;
129+
}
130+
131+
if (symbol.flags & ts.SymbolFlags.Alias) {
132+
const resolved = typeChecker.getAliasedSymbol(symbol);
133+
return containsPublicAPI(resolved);
134+
}
135+
136+
// Namespace barrel; actual namespaces are checked below.
137+
if (symbol.flags & ts.SymbolFlags.ValueModule && symbol.valueDeclaration?.kind === ts.SyntaxKind.SourceFile) {
138+
for (const me of typeChecker.getExportsOfModule(symbol)) {
139+
if (containsPublicAPI(me)) {
140+
return true;
141+
}
142+
}
143+
return false;
144+
}
145+
146+
for (const decl of symbol.declarations) {
147+
const statement = getDeclarationStatement(decl);
148+
if (statement && !isInternalDeclaration(statement)) {
149+
return true;
150+
}
151+
}
152+
153+
return false;
154+
}
155+
}
156+
157+
function nodeToLocation(decl: ts.Node): string {
158+
const sourceFile = decl.getSourceFile();
159+
const lc = sourceFile.getLineAndCharacterOfPosition(decl.pos);
160+
return `${sourceFile.fileName}:${lc.line}:${lc.character}`;
161+
}
162+
163+
function removeDeclareConstExport(node: ts.Node): ts.Node | undefined {
164+
switch (node.kind) {
165+
case ts.SyntaxKind.DeclareKeyword: // No need to emit this in d.ts files.
166+
case ts.SyntaxKind.ConstKeyword: // Remove const from const enums.
167+
case ts.SyntaxKind.ExportKeyword: // No export modifier; we are already in the namespace.
168+
return undefined;
169+
}
170+
return node;
171+
}
172+
173+
function emitAsNamespace(name: string, moduleSymbol: ts.Symbol) {
174+
assert(moduleSymbol.flags & ts.SymbolFlags.ValueModule);
175+
176+
const target = containsPublicAPI(moduleSymbol) ? WriteTarget.Both : WriteTarget.Internal;
177+
178+
if (name === "ts") {
179+
// We will write `export = ts` at the end.
180+
write(`declare namespace ${name} {`, target);
181+
}
182+
else {
183+
// No export modifier; we are already in the namespace.
184+
write(`namespace ${name} {`, target);
185+
}
186+
increaseIndent();
187+
188+
const moduleExports = typeChecker.getExportsOfModule(moduleSymbol);
189+
for (const me of moduleExports) {
190+
assert(me.declarations?.length);
191+
192+
if (me.flags & ts.SymbolFlags.Alias) {
193+
const resolved = typeChecker.getAliasedSymbol(me);
194+
emitAsNamespace(me.name, resolved);
195+
continue;
196+
}
197+
198+
for (const decl of me.declarations) {
199+
const statement = getDeclarationStatement(decl);
200+
const sourceFile = decl.getSourceFile();
201+
202+
if (!statement) {
203+
throw new Error(`Unhandled declaration for ${me.name} at ${nodeToLocation(decl)}`);
204+
}
205+
206+
const isInternal = isInternalDeclaration(statement);
207+
if (!isInternal) {
208+
const publicStatement = ts.visitEachChild(statement, (node) => {
209+
// No @internal comments in the public API.
210+
if (isInternalDeclaration(node)) {
211+
return undefined;
212+
}
213+
return removeDeclareConstExport(node);
214+
}, nullTransformationContext);
215+
216+
writeNode(publicStatement, sourceFile, WriteTarget.Public);
217+
}
218+
219+
const internalStatement = ts.visitEachChild(statement, removeDeclareConstExport, nullTransformationContext);
220+
221+
writeNode(internalStatement, sourceFile, WriteTarget.Internal);
222+
}
223+
}
224+
225+
decreaseIndent();
226+
write(`}`, target);
227+
}
228+
229+
emitAsNamespace("ts", moduleSymbol);
230+
231+
write("export = ts;", WriteTarget.Both);
232+
233+
const copyrightNotice = fs.readFileSync(path.join(__dirname, "..", "CopyrightNotice.txt"), "utf-8");
234+
const publicContents = copyrightNotice + publicLines.join(newLine);
235+
const internalContents = copyrightNotice + internalLines.join(newLine);
236+
237+
if (publicContents.includes("@internal")) {
238+
console.error("Output includes untrimmed @internal nodes!");
239+
}
240+
241+
fs.writeFileSync(output, publicContents);
242+
fs.writeFileSync(internalOutput, internalContents);

scripts/produceLKG.ts

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import * as glob from "glob";
77
const root = path.join(__dirname, "..");
88
const source = path.join(root, "built/local");
99
const dest = path.join(root, "lib");
10-
const copyright = fs.readFileSync(path.join(__dirname, "../CopyrightNotice.txt"), "utf-8");
1110

1211
async function produceLKG() {
1312
console.log(`Building LKG from ${source} to ${dest}`);
@@ -74,19 +73,14 @@ async function copyScriptOutputs() {
7473
}
7574

7675
async function copyDeclarationOutputs() {
77-
await copyWithCopyright("tsserverlibrary.d.ts");
78-
await copyWithCopyright("typescript.d.ts");
76+
await copyFromBuiltLocal("tsserverlibrary.d.ts");
77+
await copyFromBuiltLocal("typescript.d.ts");
7978
}
8079

8180
async function writeGitAttributes() {
8281
await fs.writeFile(path.join(dest, ".gitattributes"), `* text eol=lf`, "utf-8");
8382
}
8483

85-
async function copyWithCopyright(fileName: string, destName = fileName) {
86-
const content = await fs.readFile(path.join(source, fileName), "utf-8");
87-
await fs.writeFile(path.join(dest, destName), copyright + "\n" + content);
88-
}
89-
9084
async function copyFromBuiltLocal(fileName: string) {
9185
await fs.copy(path.join(source, fileName), path.join(dest, fileName));
9286
}

0 commit comments

Comments
 (0)