Compiler.js 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const parseJson = require("json-parse-better-errors");
  7. const asyncLib = require("neo-async");
  8. const {
  9. SyncHook,
  10. SyncBailHook,
  11. AsyncParallelHook,
  12. AsyncSeriesHook
  13. } = require("tapable");
  14. const { SizeOnlySource } = require("webpack-sources");
  15. const webpack = require("./");
  16. const Cache = require("./Cache");
  17. const CacheFacade = require("./CacheFacade");
  18. const ChunkGraph = require("./ChunkGraph");
  19. const Compilation = require("./Compilation");
  20. const ConcurrentCompilationError = require("./ConcurrentCompilationError");
  21. const ContextModuleFactory = require("./ContextModuleFactory");
  22. const ModuleGraph = require("./ModuleGraph");
  23. const NormalModuleFactory = require("./NormalModuleFactory");
  24. const RequestShortener = require("./RequestShortener");
  25. const ResolverFactory = require("./ResolverFactory");
  26. const Stats = require("./Stats");
  27. const Watching = require("./Watching");
  28. const WebpackError = require("./WebpackError");
  29. const { Logger } = require("./logging/Logger");
  30. const { join, dirname, mkdirp } = require("./util/fs");
  31. const { makePathsRelative } = require("./util/identifier");
  32. const { isSourceEqual } = require("./util/source");
  33. /** @typedef {import("webpack-sources").Source} Source */
  34. /** @typedef {import("../declarations/WebpackOptions").EntryNormalized} Entry */
  35. /** @typedef {import("../declarations/WebpackOptions").OutputNormalized} OutputOptions */
  36. /** @typedef {import("../declarations/WebpackOptions").WatchOptions} WatchOptions */
  37. /** @typedef {import("../declarations/WebpackOptions").WebpackOptionsNormalized} WebpackOptions */
  38. /** @typedef {import("../declarations/WebpackOptions").WebpackPluginInstance} WebpackPluginInstance */
  39. /** @typedef {import("./Chunk")} Chunk */
  40. /** @typedef {import("./Dependency")} Dependency */
  41. /** @typedef {import("./FileSystemInfo").FileSystemInfoEntry} FileSystemInfoEntry */
  42. /** @typedef {import("./Module")} Module */
  43. /** @typedef {import("./util/WeakTupleMap")} WeakTupleMap */
  44. /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
  45. /** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
  46. /** @typedef {import("./util/fs").OutputFileSystem} OutputFileSystem */
  47. /** @typedef {import("./util/fs").WatchFileSystem} WatchFileSystem */
  48. /**
  49. * @typedef {Object} CompilationParams
  50. * @property {NormalModuleFactory} normalModuleFactory
  51. * @property {ContextModuleFactory} contextModuleFactory
  52. */
  53. /**
  54. * @template T
  55. * @callback Callback
  56. * @param {(Error | null)=} err
  57. * @param {T=} result
  58. */
  59. /**
  60. * @callback RunAsChildCallback
  61. * @param {(Error | null)=} err
  62. * @param {Chunk[]=} entries
  63. * @param {Compilation=} compilation
  64. */
  65. /**
  66. * @typedef {Object} AssetEmittedInfo
  67. * @property {Buffer} content
  68. * @property {Source} source
  69. * @property {Compilation} compilation
  70. * @property {string} outputPath
  71. * @property {string} targetPath
  72. */
  73. /**
  74. * @param {string[]} array an array
  75. * @returns {boolean} true, if the array is sorted
  76. */
  77. const isSorted = array => {
  78. for (let i = 1; i < array.length; i++) {
  79. if (array[i - 1] > array[i]) return false;
  80. }
  81. return true;
  82. };
  83. /**
  84. * @param {Object} obj an object
  85. * @param {string[]} keys the keys of the object
  86. * @returns {Object} the object with properties sorted by property name
  87. */
  88. const sortObject = (obj, keys) => {
  89. const o = {};
  90. for (const k of keys.sort()) {
  91. o[k] = obj[k];
  92. }
  93. return o;
  94. };
  95. /**
  96. * @param {string} filename filename
  97. * @param {string | string[] | undefined} hashes list of hashes
  98. * @returns {boolean} true, if the filename contains any hash
  99. */
  100. const includesHash = (filename, hashes) => {
  101. if (!hashes) return false;
  102. if (Array.isArray(hashes)) {
  103. return hashes.some(hash => filename.includes(hash));
  104. } else {
  105. return filename.includes(hashes);
  106. }
  107. };
  108. class Compiler {
  109. /**
  110. * @param {string} context the compilation path
  111. * @param {WebpackOptions} options options
  112. */
  113. constructor(context, options = /** @type {WebpackOptions} */ ({})) {
  114. this.hooks = Object.freeze({
  115. /** @type {SyncHook<[]>} */
  116. initialize: new SyncHook([]),
  117. /** @type {SyncBailHook<[Compilation], boolean>} */
  118. shouldEmit: new SyncBailHook(["compilation"]),
  119. /** @type {AsyncSeriesHook<[Stats]>} */
  120. done: new AsyncSeriesHook(["stats"]),
  121. /** @type {SyncHook<[Stats]>} */
  122. afterDone: new SyncHook(["stats"]),
  123. /** @type {AsyncSeriesHook<[]>} */
  124. additionalPass: new AsyncSeriesHook([]),
  125. /** @type {AsyncSeriesHook<[Compiler]>} */
  126. beforeRun: new AsyncSeriesHook(["compiler"]),
  127. /** @type {AsyncSeriesHook<[Compiler]>} */
  128. run: new AsyncSeriesHook(["compiler"]),
  129. /** @type {AsyncSeriesHook<[Compilation]>} */
  130. emit: new AsyncSeriesHook(["compilation"]),
  131. /** @type {AsyncSeriesHook<[string, AssetEmittedInfo]>} */
  132. assetEmitted: new AsyncSeriesHook(["file", "info"]),
  133. /** @type {AsyncSeriesHook<[Compilation]>} */
  134. afterEmit: new AsyncSeriesHook(["compilation"]),
  135. /** @type {SyncHook<[Compilation, CompilationParams]>} */
  136. thisCompilation: new SyncHook(["compilation", "params"]),
  137. /** @type {SyncHook<[Compilation, CompilationParams]>} */
  138. compilation: new SyncHook(["compilation", "params"]),
  139. /** @type {SyncHook<[NormalModuleFactory]>} */
  140. normalModuleFactory: new SyncHook(["normalModuleFactory"]),
  141. /** @type {SyncHook<[ContextModuleFactory]>} */
  142. contextModuleFactory: new SyncHook(["contextModuleFactory"]),
  143. /** @type {AsyncSeriesHook<[CompilationParams]>} */
  144. beforeCompile: new AsyncSeriesHook(["params"]),
  145. /** @type {SyncHook<[CompilationParams]>} */
  146. compile: new SyncHook(["params"]),
  147. /** @type {AsyncParallelHook<[Compilation]>} */
  148. make: new AsyncParallelHook(["compilation"]),
  149. /** @type {AsyncParallelHook<[Compilation]>} */
  150. finishMake: new AsyncSeriesHook(["compilation"]),
  151. /** @type {AsyncSeriesHook<[Compilation]>} */
  152. afterCompile: new AsyncSeriesHook(["compilation"]),
  153. /** @type {AsyncSeriesHook<[]>} */
  154. readRecords: new AsyncSeriesHook([]),
  155. /** @type {AsyncSeriesHook<[]>} */
  156. emitRecords: new AsyncSeriesHook([]),
  157. /** @type {AsyncSeriesHook<[Compiler]>} */
  158. watchRun: new AsyncSeriesHook(["compiler"]),
  159. /** @type {SyncHook<[Error]>} */
  160. failed: new SyncHook(["error"]),
  161. /** @type {SyncHook<[string | null, number]>} */
  162. invalid: new SyncHook(["filename", "changeTime"]),
  163. /** @type {SyncHook<[]>} */
  164. watchClose: new SyncHook([]),
  165. /** @type {AsyncSeriesHook<[]>} */
  166. shutdown: new AsyncSeriesHook([]),
  167. /** @type {SyncBailHook<[string, string, any[]], true>} */
  168. infrastructureLog: new SyncBailHook(["origin", "type", "args"]),
  169. // TODO the following hooks are weirdly located here
  170. // TODO move them for webpack 5
  171. /** @type {SyncHook<[]>} */
  172. environment: new SyncHook([]),
  173. /** @type {SyncHook<[]>} */
  174. afterEnvironment: new SyncHook([]),
  175. /** @type {SyncHook<[Compiler]>} */
  176. afterPlugins: new SyncHook(["compiler"]),
  177. /** @type {SyncHook<[Compiler]>} */
  178. afterResolvers: new SyncHook(["compiler"]),
  179. /** @type {SyncBailHook<[string, Entry], boolean>} */
  180. entryOption: new SyncBailHook(["context", "entry"])
  181. });
  182. this.webpack = webpack;
  183. /** @type {string=} */
  184. this.name = undefined;
  185. /** @type {Compilation=} */
  186. this.parentCompilation = undefined;
  187. /** @type {Compiler} */
  188. this.root = this;
  189. /** @type {string} */
  190. this.outputPath = "";
  191. /** @type {Watching} */
  192. this.watching = undefined;
  193. /** @type {OutputFileSystem} */
  194. this.outputFileSystem = null;
  195. /** @type {IntermediateFileSystem} */
  196. this.intermediateFileSystem = null;
  197. /** @type {InputFileSystem} */
  198. this.inputFileSystem = null;
  199. /** @type {WatchFileSystem} */
  200. this.watchFileSystem = null;
  201. /** @type {string|null} */
  202. this.recordsInputPath = null;
  203. /** @type {string|null} */
  204. this.recordsOutputPath = null;
  205. this.records = {};
  206. /** @type {Set<string | RegExp>} */
  207. this.managedPaths = new Set();
  208. /** @type {Set<string | RegExp>} */
  209. this.immutablePaths = new Set();
  210. /** @type {ReadonlySet<string>} */
  211. this.modifiedFiles = undefined;
  212. /** @type {ReadonlySet<string>} */
  213. this.removedFiles = undefined;
  214. /** @type {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null>} */
  215. this.fileTimestamps = undefined;
  216. /** @type {ReadonlyMap<string, FileSystemInfoEntry | "ignore" | null>} */
  217. this.contextTimestamps = undefined;
  218. /** @type {number} */
  219. this.fsStartTime = undefined;
  220. /** @type {ResolverFactory} */
  221. this.resolverFactory = new ResolverFactory();
  222. this.infrastructureLogger = undefined;
  223. this.options = options;
  224. this.context = context;
  225. this.requestShortener = new RequestShortener(context, this.root);
  226. this.cache = new Cache();
  227. /** @type {Map<Module, { buildInfo: object, references: WeakMap<Dependency, Module>, memCache: WeakTupleMap }> | undefined} */
  228. this.moduleMemCaches = undefined;
  229. this.compilerPath = "";
  230. /** @type {boolean} */
  231. this.running = false;
  232. /** @type {boolean} */
  233. this.idle = false;
  234. /** @type {boolean} */
  235. this.watchMode = false;
  236. this._backCompat = this.options.experiments.backCompat !== false;
  237. /** @type {Compilation} */
  238. this._lastCompilation = undefined;
  239. /** @type {NormalModuleFactory} */
  240. this._lastNormalModuleFactory = undefined;
  241. /** @private @type {WeakMap<Source, { sizeOnlySource: SizeOnlySource, writtenTo: Map<string, number> }>} */
  242. this._assetEmittingSourceCache = new WeakMap();
  243. /** @private @type {Map<string, number>} */
  244. this._assetEmittingWrittenFiles = new Map();
  245. /** @private @type {Set<string>} */
  246. this._assetEmittingPreviousFiles = new Set();
  247. }
  248. /**
  249. * @param {string} name cache name
  250. * @returns {CacheFacade} the cache facade instance
  251. */
  252. getCache(name) {
  253. return new CacheFacade(
  254. this.cache,
  255. `${this.compilerPath}${name}`,
  256. this.options.output.hashFunction
  257. );
  258. }
  259. /**
  260. * @param {string | (function(): string)} name name of the logger, or function called once to get the logger name
  261. * @returns {Logger} a logger with that name
  262. */
  263. getInfrastructureLogger(name) {
  264. if (!name) {
  265. throw new TypeError(
  266. "Compiler.getInfrastructureLogger(name) called without a name"
  267. );
  268. }
  269. return new Logger(
  270. (type, args) => {
  271. if (typeof name === "function") {
  272. name = name();
  273. if (!name) {
  274. throw new TypeError(
  275. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  276. );
  277. }
  278. }
  279. if (this.hooks.infrastructureLog.call(name, type, args) === undefined) {
  280. if (this.infrastructureLogger !== undefined) {
  281. this.infrastructureLogger(name, type, args);
  282. }
  283. }
  284. },
  285. childName => {
  286. if (typeof name === "function") {
  287. if (typeof childName === "function") {
  288. return this.getInfrastructureLogger(() => {
  289. if (typeof name === "function") {
  290. name = name();
  291. if (!name) {
  292. throw new TypeError(
  293. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  294. );
  295. }
  296. }
  297. if (typeof childName === "function") {
  298. childName = childName();
  299. if (!childName) {
  300. throw new TypeError(
  301. "Logger.getChildLogger(name) called with a function not returning a name"
  302. );
  303. }
  304. }
  305. return `${name}/${childName}`;
  306. });
  307. } else {
  308. return this.getInfrastructureLogger(() => {
  309. if (typeof name === "function") {
  310. name = name();
  311. if (!name) {
  312. throw new TypeError(
  313. "Compiler.getInfrastructureLogger(name) called with a function not returning a name"
  314. );
  315. }
  316. }
  317. return `${name}/${childName}`;
  318. });
  319. }
  320. } else {
  321. if (typeof childName === "function") {
  322. return this.getInfrastructureLogger(() => {
  323. if (typeof childName === "function") {
  324. childName = childName();
  325. if (!childName) {
  326. throw new TypeError(
  327. "Logger.getChildLogger(name) called with a function not returning a name"
  328. );
  329. }
  330. }
  331. return `${name}/${childName}`;
  332. });
  333. } else {
  334. return this.getInfrastructureLogger(`${name}/${childName}`);
  335. }
  336. }
  337. }
  338. );
  339. }
  340. // TODO webpack 6: solve this in a better way
  341. // e.g. move compilation specific info from Modules into ModuleGraph
  342. _cleanupLastCompilation() {
  343. if (this._lastCompilation !== undefined) {
  344. for (const module of this._lastCompilation.modules) {
  345. ChunkGraph.clearChunkGraphForModule(module);
  346. ModuleGraph.clearModuleGraphForModule(module);
  347. module.cleanupForCache();
  348. }
  349. for (const chunk of this._lastCompilation.chunks) {
  350. ChunkGraph.clearChunkGraphForChunk(chunk);
  351. }
  352. this._lastCompilation = undefined;
  353. }
  354. }
  355. // TODO webpack 6: solve this in a better way
  356. _cleanupLastNormalModuleFactory() {
  357. if (this._lastNormalModuleFactory !== undefined) {
  358. this._lastNormalModuleFactory.cleanupForCache();
  359. this._lastNormalModuleFactory = undefined;
  360. }
  361. }
  362. /**
  363. * @param {WatchOptions} watchOptions the watcher's options
  364. * @param {Callback<Stats>} handler signals when the call finishes
  365. * @returns {Watching} a compiler watcher
  366. */
  367. watch(watchOptions, handler) {
  368. if (this.running) {
  369. return handler(new ConcurrentCompilationError());
  370. }
  371. this.running = true;
  372. this.watchMode = true;
  373. this.watching = new Watching(this, watchOptions, handler);
  374. return this.watching;
  375. }
  376. /**
  377. * @param {Callback<Stats>} callback signals when the call finishes
  378. * @returns {void}
  379. */
  380. run(callback) {
  381. if (this.running) {
  382. return callback(new ConcurrentCompilationError());
  383. }
  384. let logger;
  385. const finalCallback = (err, stats) => {
  386. if (logger) logger.time("beginIdle");
  387. this.idle = true;
  388. this.cache.beginIdle();
  389. this.idle = true;
  390. if (logger) logger.timeEnd("beginIdle");
  391. this.running = false;
  392. if (err) {
  393. this.hooks.failed.call(err);
  394. }
  395. if (callback !== undefined) callback(err, stats);
  396. this.hooks.afterDone.call(stats);
  397. };
  398. const startTime = Date.now();
  399. this.running = true;
  400. const onCompiled = (err, compilation) => {
  401. if (err) return finalCallback(err);
  402. if (this.hooks.shouldEmit.call(compilation) === false) {
  403. compilation.startTime = startTime;
  404. compilation.endTime = Date.now();
  405. const stats = new Stats(compilation);
  406. this.hooks.done.callAsync(stats, err => {
  407. if (err) return finalCallback(err);
  408. return finalCallback(null, stats);
  409. });
  410. return;
  411. }
  412. process.nextTick(() => {
  413. logger = compilation.getLogger("webpack.Compiler");
  414. logger.time("emitAssets");
  415. this.emitAssets(compilation, err => {
  416. logger.timeEnd("emitAssets");
  417. if (err) return finalCallback(err);
  418. if (compilation.hooks.needAdditionalPass.call()) {
  419. compilation.needAdditionalPass = true;
  420. compilation.startTime = startTime;
  421. compilation.endTime = Date.now();
  422. logger.time("done hook");
  423. const stats = new Stats(compilation);
  424. this.hooks.done.callAsync(stats, err => {
  425. logger.timeEnd("done hook");
  426. if (err) return finalCallback(err);
  427. this.hooks.additionalPass.callAsync(err => {
  428. if (err) return finalCallback(err);
  429. this.compile(onCompiled);
  430. });
  431. });
  432. return;
  433. }
  434. logger.time("emitRecords");
  435. this.emitRecords(err => {
  436. logger.timeEnd("emitRecords");
  437. if (err) return finalCallback(err);
  438. compilation.startTime = startTime;
  439. compilation.endTime = Date.now();
  440. logger.time("done hook");
  441. const stats = new Stats(compilation);
  442. this.hooks.done.callAsync(stats, err => {
  443. logger.timeEnd("done hook");
  444. if (err) return finalCallback(err);
  445. this.cache.storeBuildDependencies(
  446. compilation.buildDependencies,
  447. err => {
  448. if (err) return finalCallback(err);
  449. return finalCallback(null, stats);
  450. }
  451. );
  452. });
  453. });
  454. });
  455. });
  456. };
  457. const run = () => {
  458. this.hooks.beforeRun.callAsync(this, err => {
  459. if (err) return finalCallback(err);
  460. this.hooks.run.callAsync(this, err => {
  461. if (err) return finalCallback(err);
  462. this.readRecords(err => {
  463. if (err) return finalCallback(err);
  464. this.compile(onCompiled);
  465. });
  466. });
  467. });
  468. };
  469. if (this.idle) {
  470. this.cache.endIdle(err => {
  471. if (err) return finalCallback(err);
  472. this.idle = false;
  473. run();
  474. });
  475. } else {
  476. run();
  477. }
  478. }
  479. /**
  480. * @param {RunAsChildCallback} callback signals when the call finishes
  481. * @returns {void}
  482. */
  483. runAsChild(callback) {
  484. const startTime = Date.now();
  485. this.compile((err, compilation) => {
  486. if (err) return callback(err);
  487. this.parentCompilation.children.push(compilation);
  488. for (const { name, source, info } of compilation.getAssets()) {
  489. this.parentCompilation.emitAsset(name, source, info);
  490. }
  491. const entries = [];
  492. for (const ep of compilation.entrypoints.values()) {
  493. entries.push(...ep.chunks);
  494. }
  495. compilation.startTime = startTime;
  496. compilation.endTime = Date.now();
  497. return callback(null, entries, compilation);
  498. });
  499. }
  500. purgeInputFileSystem() {
  501. if (this.inputFileSystem && this.inputFileSystem.purge) {
  502. this.inputFileSystem.purge();
  503. }
  504. }
  505. /**
  506. * @param {Compilation} compilation the compilation
  507. * @param {Callback<void>} callback signals when the assets are emitted
  508. * @returns {void}
  509. */
  510. emitAssets(compilation, callback) {
  511. let outputPath;
  512. const emitFiles = err => {
  513. if (err) return callback(err);
  514. const assets = compilation.getAssets();
  515. compilation.assets = { ...compilation.assets };
  516. /** @type {Map<string, { path: string, source: Source, size: number, waiting: { cacheEntry: any, file: string }[] }>} */
  517. const caseInsensitiveMap = new Map();
  518. /** @type {Set<string>} */
  519. const allTargetPaths = new Set();
  520. asyncLib.forEachLimit(
  521. assets,
  522. 15,
  523. ({ name: file, source, info }, callback) => {
  524. let targetFile = file;
  525. let immutable = info.immutable;
  526. const queryStringIdx = targetFile.indexOf("?");
  527. if (queryStringIdx >= 0) {
  528. targetFile = targetFile.substr(0, queryStringIdx);
  529. // We may remove the hash, which is in the query string
  530. // So we recheck if the file is immutable
  531. // This doesn't cover all cases, but immutable is only a performance optimization anyway
  532. immutable =
  533. immutable &&
  534. (includesHash(targetFile, info.contenthash) ||
  535. includesHash(targetFile, info.chunkhash) ||
  536. includesHash(targetFile, info.modulehash) ||
  537. includesHash(targetFile, info.fullhash));
  538. }
  539. const writeOut = err => {
  540. if (err) return callback(err);
  541. const targetPath = join(
  542. this.outputFileSystem,
  543. outputPath,
  544. targetFile
  545. );
  546. allTargetPaths.add(targetPath);
  547. // check if the target file has already been written by this Compiler
  548. const targetFileGeneration =
  549. this._assetEmittingWrittenFiles.get(targetPath);
  550. // create an cache entry for this Source if not already existing
  551. let cacheEntry = this._assetEmittingSourceCache.get(source);
  552. if (cacheEntry === undefined) {
  553. cacheEntry = {
  554. sizeOnlySource: undefined,
  555. writtenTo: new Map()
  556. };
  557. this._assetEmittingSourceCache.set(source, cacheEntry);
  558. }
  559. let similarEntry;
  560. const checkSimilarFile = () => {
  561. const caseInsensitiveTargetPath = targetPath.toLowerCase();
  562. similarEntry = caseInsensitiveMap.get(caseInsensitiveTargetPath);
  563. if (similarEntry !== undefined) {
  564. const { path: other, source: otherSource } = similarEntry;
  565. if (isSourceEqual(otherSource, source)) {
  566. // Size may or may not be available at this point.
  567. // If it's not available add to "waiting" list and it will be updated once available
  568. if (similarEntry.size !== undefined) {
  569. updateWithReplacementSource(similarEntry.size);
  570. } else {
  571. if (!similarEntry.waiting) similarEntry.waiting = [];
  572. similarEntry.waiting.push({ file, cacheEntry });
  573. }
  574. alreadyWritten();
  575. } else {
  576. const err =
  577. new WebpackError(`Prevent writing to file that only differs in casing or query string from already written file.
  578. This will lead to a race-condition and corrupted files on case-insensitive file systems.
  579. ${targetPath}
  580. ${other}`);
  581. err.file = file;
  582. callback(err);
  583. }
  584. return true;
  585. } else {
  586. caseInsensitiveMap.set(
  587. caseInsensitiveTargetPath,
  588. (similarEntry = {
  589. path: targetPath,
  590. source,
  591. size: undefined,
  592. waiting: undefined
  593. })
  594. );
  595. return false;
  596. }
  597. };
  598. /**
  599. * get the binary (Buffer) content from the Source
  600. * @returns {Buffer} content for the source
  601. */
  602. const getContent = () => {
  603. if (typeof source.buffer === "function") {
  604. return source.buffer();
  605. } else {
  606. const bufferOrString = source.source();
  607. if (Buffer.isBuffer(bufferOrString)) {
  608. return bufferOrString;
  609. } else {
  610. return Buffer.from(bufferOrString, "utf8");
  611. }
  612. }
  613. };
  614. const alreadyWritten = () => {
  615. // cache the information that the Source has been already been written to that location
  616. if (targetFileGeneration === undefined) {
  617. const newGeneration = 1;
  618. this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
  619. cacheEntry.writtenTo.set(targetPath, newGeneration);
  620. } else {
  621. cacheEntry.writtenTo.set(targetPath, targetFileGeneration);
  622. }
  623. callback();
  624. };
  625. /**
  626. * Write the file to output file system
  627. * @param {Buffer} content content to be written
  628. * @returns {void}
  629. */
  630. const doWrite = content => {
  631. this.outputFileSystem.writeFile(targetPath, content, err => {
  632. if (err) return callback(err);
  633. // information marker that the asset has been emitted
  634. compilation.emittedAssets.add(file);
  635. // cache the information that the Source has been written to that location
  636. const newGeneration =
  637. targetFileGeneration === undefined
  638. ? 1
  639. : targetFileGeneration + 1;
  640. cacheEntry.writtenTo.set(targetPath, newGeneration);
  641. this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
  642. this.hooks.assetEmitted.callAsync(
  643. file,
  644. {
  645. content,
  646. source,
  647. outputPath,
  648. compilation,
  649. targetPath
  650. },
  651. callback
  652. );
  653. });
  654. };
  655. const updateWithReplacementSource = size => {
  656. updateFileWithReplacementSource(file, cacheEntry, size);
  657. similarEntry.size = size;
  658. if (similarEntry.waiting !== undefined) {
  659. for (const { file, cacheEntry } of similarEntry.waiting) {
  660. updateFileWithReplacementSource(file, cacheEntry, size);
  661. }
  662. }
  663. };
  664. const updateFileWithReplacementSource = (
  665. file,
  666. cacheEntry,
  667. size
  668. ) => {
  669. // Create a replacement resource which only allows to ask for size
  670. // This allows to GC all memory allocated by the Source
  671. // (expect when the Source is stored in any other cache)
  672. if (!cacheEntry.sizeOnlySource) {
  673. cacheEntry.sizeOnlySource = new SizeOnlySource(size);
  674. }
  675. compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
  676. size
  677. });
  678. };
  679. const processExistingFile = stats => {
  680. // skip emitting if it's already there and an immutable file
  681. if (immutable) {
  682. updateWithReplacementSource(stats.size);
  683. return alreadyWritten();
  684. }
  685. const content = getContent();
  686. updateWithReplacementSource(content.length);
  687. // if it exists and content on disk matches content
  688. // skip writing the same content again
  689. // (to keep mtime and don't trigger watchers)
  690. // for a fast negative match file size is compared first
  691. if (content.length === stats.size) {
  692. compilation.comparedForEmitAssets.add(file);
  693. return this.outputFileSystem.readFile(
  694. targetPath,
  695. (err, existingContent) => {
  696. if (
  697. err ||
  698. !content.equals(/** @type {Buffer} */ (existingContent))
  699. ) {
  700. return doWrite(content);
  701. } else {
  702. return alreadyWritten();
  703. }
  704. }
  705. );
  706. }
  707. return doWrite(content);
  708. };
  709. const processMissingFile = () => {
  710. const content = getContent();
  711. updateWithReplacementSource(content.length);
  712. return doWrite(content);
  713. };
  714. // if the target file has already been written
  715. if (targetFileGeneration !== undefined) {
  716. // check if the Source has been written to this target file
  717. const writtenGeneration = cacheEntry.writtenTo.get(targetPath);
  718. if (writtenGeneration === targetFileGeneration) {
  719. // if yes, we may skip writing the file
  720. // if it's already there
  721. // (we assume one doesn't modify files while the Compiler is running, other then removing them)
  722. if (this._assetEmittingPreviousFiles.has(targetPath)) {
  723. // We assume that assets from the last compilation say intact on disk (they are not removed)
  724. compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
  725. size: cacheEntry.sizeOnlySource.size()
  726. });
  727. return callback();
  728. } else {
  729. // Settings immutable will make it accept file content without comparing when file exist
  730. immutable = true;
  731. }
  732. } else if (!immutable) {
  733. if (checkSimilarFile()) return;
  734. // We wrote to this file before which has very likely a different content
  735. // skip comparing and assume content is different for performance
  736. // This case happens often during watch mode.
  737. return processMissingFile();
  738. }
  739. }
  740. if (checkSimilarFile()) return;
  741. if (this.options.output.compareBeforeEmit) {
  742. this.outputFileSystem.stat(targetPath, (err, stats) => {
  743. const exists = !err && stats.isFile();
  744. if (exists) {
  745. processExistingFile(stats);
  746. } else {
  747. processMissingFile();
  748. }
  749. });
  750. } else {
  751. processMissingFile();
  752. }
  753. };
  754. if (targetFile.match(/\/|\\/)) {
  755. const fs = this.outputFileSystem;
  756. const dir = dirname(fs, join(fs, outputPath, targetFile));
  757. mkdirp(fs, dir, writeOut);
  758. } else {
  759. writeOut();
  760. }
  761. },
  762. err => {
  763. // Clear map to free up memory
  764. caseInsensitiveMap.clear();
  765. if (err) {
  766. this._assetEmittingPreviousFiles.clear();
  767. return callback(err);
  768. }
  769. this._assetEmittingPreviousFiles = allTargetPaths;
  770. this.hooks.afterEmit.callAsync(compilation, err => {
  771. if (err) return callback(err);
  772. return callback();
  773. });
  774. }
  775. );
  776. };
  777. this.hooks.emit.callAsync(compilation, err => {
  778. if (err) return callback(err);
  779. outputPath = compilation.getPath(this.outputPath, {});
  780. mkdirp(this.outputFileSystem, outputPath, emitFiles);
  781. });
  782. }
  783. /**
  784. * @param {Callback<void>} callback signals when the call finishes
  785. * @returns {void}
  786. */
  787. emitRecords(callback) {
  788. if (this.hooks.emitRecords.isUsed()) {
  789. if (this.recordsOutputPath) {
  790. asyncLib.parallel(
  791. [
  792. cb => this.hooks.emitRecords.callAsync(cb),
  793. this._emitRecords.bind(this)
  794. ],
  795. err => callback(err)
  796. );
  797. } else {
  798. this.hooks.emitRecords.callAsync(callback);
  799. }
  800. } else {
  801. if (this.recordsOutputPath) {
  802. this._emitRecords(callback);
  803. } else {
  804. callback();
  805. }
  806. }
  807. }
  808. /**
  809. * @param {Callback<void>} callback signals when the call finishes
  810. * @returns {void}
  811. */
  812. _emitRecords(callback) {
  813. const writeFile = () => {
  814. this.outputFileSystem.writeFile(
  815. this.recordsOutputPath,
  816. JSON.stringify(
  817. this.records,
  818. (n, value) => {
  819. if (
  820. typeof value === "object" &&
  821. value !== null &&
  822. !Array.isArray(value)
  823. ) {
  824. const keys = Object.keys(value);
  825. if (!isSorted(keys)) {
  826. return sortObject(value, keys);
  827. }
  828. }
  829. return value;
  830. },
  831. 2
  832. ),
  833. callback
  834. );
  835. };
  836. const recordsOutputPathDirectory = dirname(
  837. this.outputFileSystem,
  838. this.recordsOutputPath
  839. );
  840. if (!recordsOutputPathDirectory) {
  841. return writeFile();
  842. }
  843. mkdirp(this.outputFileSystem, recordsOutputPathDirectory, err => {
  844. if (err) return callback(err);
  845. writeFile();
  846. });
  847. }
  848. /**
  849. * @param {Callback<void>} callback signals when the call finishes
  850. * @returns {void}
  851. */
  852. readRecords(callback) {
  853. if (this.hooks.readRecords.isUsed()) {
  854. if (this.recordsInputPath) {
  855. asyncLib.parallel([
  856. cb => this.hooks.readRecords.callAsync(cb),
  857. this._readRecords.bind(this)
  858. ]);
  859. } else {
  860. this.records = {};
  861. this.hooks.readRecords.callAsync(callback);
  862. }
  863. } else {
  864. if (this.recordsInputPath) {
  865. this._readRecords(callback);
  866. } else {
  867. this.records = {};
  868. callback();
  869. }
  870. }
  871. }
  872. /**
  873. * @param {Callback<void>} callback signals when the call finishes
  874. * @returns {void}
  875. */
  876. _readRecords(callback) {
  877. if (!this.recordsInputPath) {
  878. this.records = {};
  879. return callback();
  880. }
  881. this.inputFileSystem.stat(this.recordsInputPath, err => {
  882. // It doesn't exist
  883. // We can ignore this.
  884. if (err) return callback();
  885. this.inputFileSystem.readFile(this.recordsInputPath, (err, content) => {
  886. if (err) return callback(err);
  887. try {
  888. this.records = parseJson(content.toString("utf-8"));
  889. } catch (e) {
  890. e.message = "Cannot parse records: " + e.message;
  891. return callback(e);
  892. }
  893. return callback();
  894. });
  895. });
  896. }
  897. /**
  898. * @param {Compilation} compilation the compilation
  899. * @param {string} compilerName the compiler's name
  900. * @param {number} compilerIndex the compiler's index
  901. * @param {OutputOptions=} outputOptions the output options
  902. * @param {WebpackPluginInstance[]=} plugins the plugins to apply
  903. * @returns {Compiler} a child compiler
  904. */
  905. createChildCompiler(
  906. compilation,
  907. compilerName,
  908. compilerIndex,
  909. outputOptions,
  910. plugins
  911. ) {
  912. const childCompiler = new Compiler(this.context, {
  913. ...this.options,
  914. output: {
  915. ...this.options.output,
  916. ...outputOptions
  917. }
  918. });
  919. childCompiler.name = compilerName;
  920. childCompiler.outputPath = this.outputPath;
  921. childCompiler.inputFileSystem = this.inputFileSystem;
  922. childCompiler.outputFileSystem = null;
  923. childCompiler.resolverFactory = this.resolverFactory;
  924. childCompiler.modifiedFiles = this.modifiedFiles;
  925. childCompiler.removedFiles = this.removedFiles;
  926. childCompiler.fileTimestamps = this.fileTimestamps;
  927. childCompiler.contextTimestamps = this.contextTimestamps;
  928. childCompiler.fsStartTime = this.fsStartTime;
  929. childCompiler.cache = this.cache;
  930. childCompiler.compilerPath = `${this.compilerPath}${compilerName}|${compilerIndex}|`;
  931. childCompiler._backCompat = this._backCompat;
  932. const relativeCompilerName = makePathsRelative(
  933. this.context,
  934. compilerName,
  935. this.root
  936. );
  937. if (!this.records[relativeCompilerName]) {
  938. this.records[relativeCompilerName] = [];
  939. }
  940. if (this.records[relativeCompilerName][compilerIndex]) {
  941. childCompiler.records = this.records[relativeCompilerName][compilerIndex];
  942. } else {
  943. this.records[relativeCompilerName].push((childCompiler.records = {}));
  944. }
  945. childCompiler.parentCompilation = compilation;
  946. childCompiler.root = this.root;
  947. if (Array.isArray(plugins)) {
  948. for (const plugin of plugins) {
  949. plugin.apply(childCompiler);
  950. }
  951. }
  952. for (const name in this.hooks) {
  953. if (
  954. ![
  955. "make",
  956. "compile",
  957. "emit",
  958. "afterEmit",
  959. "invalid",
  960. "done",
  961. "thisCompilation"
  962. ].includes(name)
  963. ) {
  964. if (childCompiler.hooks[name]) {
  965. childCompiler.hooks[name].taps = this.hooks[name].taps.slice();
  966. }
  967. }
  968. }
  969. compilation.hooks.childCompiler.call(
  970. childCompiler,
  971. compilerName,
  972. compilerIndex
  973. );
  974. return childCompiler;
  975. }
  976. isChild() {
  977. return !!this.parentCompilation;
  978. }
  979. createCompilation(params) {
  980. this._cleanupLastCompilation();
  981. return (this._lastCompilation = new Compilation(this, params));
  982. }
  983. /**
  984. * @param {CompilationParams} params the compilation parameters
  985. * @returns {Compilation} the created compilation
  986. */
  987. newCompilation(params) {
  988. const compilation = this.createCompilation(params);
  989. compilation.name = this.name;
  990. compilation.records = this.records;
  991. this.hooks.thisCompilation.call(compilation, params);
  992. this.hooks.compilation.call(compilation, params);
  993. return compilation;
  994. }
  995. createNormalModuleFactory() {
  996. this._cleanupLastNormalModuleFactory();
  997. const normalModuleFactory = new NormalModuleFactory({
  998. context: this.options.context,
  999. fs: this.inputFileSystem,
  1000. resolverFactory: this.resolverFactory,
  1001. options: this.options.module,
  1002. associatedObjectForCache: this.root,
  1003. layers: this.options.experiments.layers
  1004. });
  1005. this._lastNormalModuleFactory = normalModuleFactory;
  1006. this.hooks.normalModuleFactory.call(normalModuleFactory);
  1007. return normalModuleFactory;
  1008. }
  1009. createContextModuleFactory() {
  1010. const contextModuleFactory = new ContextModuleFactory(this.resolverFactory);
  1011. this.hooks.contextModuleFactory.call(contextModuleFactory);
  1012. return contextModuleFactory;
  1013. }
  1014. newCompilationParams() {
  1015. const params = {
  1016. normalModuleFactory: this.createNormalModuleFactory(),
  1017. contextModuleFactory: this.createContextModuleFactory()
  1018. };
  1019. return params;
  1020. }
  1021. /**
  1022. * @param {Callback<Compilation>} callback signals when the compilation finishes
  1023. * @returns {void}
  1024. */
  1025. compile(callback) {
  1026. const params = this.newCompilationParams();
  1027. this.hooks.beforeCompile.callAsync(params, err => {
  1028. if (err) return callback(err);
  1029. this.hooks.compile.call(params);
  1030. const compilation = this.newCompilation(params);
  1031. const logger = compilation.getLogger("webpack.Compiler");
  1032. logger.time("make hook");
  1033. this.hooks.make.callAsync(compilation, err => {
  1034. logger.timeEnd("make hook");
  1035. if (err) return callback(err);
  1036. logger.time("finish make hook");
  1037. this.hooks.finishMake.callAsync(compilation, err => {
  1038. logger.timeEnd("finish make hook");
  1039. if (err) return callback(err);
  1040. process.nextTick(() => {
  1041. logger.time("finish compilation");
  1042. compilation.finish(err => {
  1043. logger.timeEnd("finish compilation");
  1044. if (err) return callback(err);
  1045. logger.time("seal compilation");
  1046. compilation.seal(err => {
  1047. logger.timeEnd("seal compilation");
  1048. if (err) return callback(err);
  1049. logger.time("afterCompile hook");
  1050. this.hooks.afterCompile.callAsync(compilation, err => {
  1051. logger.timeEnd("afterCompile hook");
  1052. if (err) return callback(err);
  1053. return callback(null, compilation);
  1054. });
  1055. });
  1056. });
  1057. });
  1058. });
  1059. });
  1060. });
  1061. }
  1062. /**
  1063. * @param {Callback<void>} callback signals when the compiler closes
  1064. * @returns {void}
  1065. */
  1066. close(callback) {
  1067. if (this.watching) {
  1068. // When there is still an active watching, close this first
  1069. this.watching.close(err => {
  1070. this.close(callback);
  1071. });
  1072. return;
  1073. }
  1074. this.hooks.shutdown.callAsync(err => {
  1075. if (err) return callback(err);
  1076. // Get rid of reference to last compilation to avoid leaking memory
  1077. // We can't run this._cleanupLastCompilation() as the Stats to this compilation
  1078. // might be still in use. We try to get rid of the reference to the cache instead.
  1079. this._lastCompilation = undefined;
  1080. this._lastNormalModuleFactory = undefined;
  1081. this.cache.shutdown(callback);
  1082. });
  1083. }
  1084. }
  1085. module.exports = Compiler;