HttpUriPlugin.js 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const { extname, basename } = require("path");
  7. const { URL } = require("url");
  8. const { createGunzip, createBrotliDecompress, createInflate } = require("zlib");
  9. const NormalModule = require("../NormalModule");
  10. const createSchemaValidation = require("../util/create-schema-validation");
  11. const createHash = require("../util/createHash");
  12. const { mkdirp, dirname, join } = require("../util/fs");
  13. const memoize = require("../util/memoize");
  14. /** @typedef {import("../../declarations/plugins/schemes/HttpUriPlugin").HttpUriPluginOptions} HttpUriPluginOptions */
  15. /** @typedef {import("../Compiler")} Compiler */
  16. const getHttp = memoize(() => require("http"));
  17. const getHttps = memoize(() => require("https"));
  18. /** @type {(() => void)[] | undefined} */
  19. let inProgressWrite = undefined;
  20. const validate = createSchemaValidation(
  21. require("../../schemas/plugins/schemes/HttpUriPlugin.check.js"),
  22. () => require("../../schemas/plugins/schemes/HttpUriPlugin.json"),
  23. {
  24. name: "Http Uri Plugin",
  25. baseDataPath: "options"
  26. }
  27. );
  28. const toSafePath = str =>
  29. str
  30. .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, "")
  31. .replace(/[^a-zA-Z0-9._-]+/g, "_");
  32. const computeIntegrity = content => {
  33. const hash = createHash("sha512");
  34. hash.update(content);
  35. const integrity = "sha512-" + hash.digest("base64");
  36. return integrity;
  37. };
  38. const verifyIntegrity = (content, integrity) => {
  39. if (integrity === "ignore") return true;
  40. return computeIntegrity(content) === integrity;
  41. };
  42. /**
  43. * @param {string} str input
  44. * @returns {Record<string, string>} parsed
  45. */
  46. const parseKeyValuePairs = str => {
  47. /** @type {Record<string, string>} */
  48. const result = {};
  49. for (const item of str.split(",")) {
  50. const i = item.indexOf("=");
  51. if (i >= 0) {
  52. const key = item.slice(0, i).trim();
  53. const value = item.slice(i + 1).trim();
  54. result[key] = value;
  55. } else {
  56. const key = item.trim();
  57. if (!key) continue;
  58. result[key] = key;
  59. }
  60. }
  61. return result;
  62. };
  63. const parseCacheControl = (cacheControl, requestTime) => {
  64. // When false resource is not stored in cache
  65. let storeCache = true;
  66. // When false resource is not stored in lockfile cache
  67. let storeLock = true;
  68. // Resource is only revalidated, after that timestamp and when upgrade is chosen
  69. let validUntil = 0;
  70. if (cacheControl) {
  71. const parsed = parseKeyValuePairs(cacheControl);
  72. if (parsed["no-cache"]) storeCache = storeLock = false;
  73. if (parsed["max-age"] && !isNaN(+parsed["max-age"])) {
  74. validUntil = requestTime + +parsed["max-age"] * 1000;
  75. }
  76. if (parsed["must-revalidate"]) validUntil = 0;
  77. }
  78. return {
  79. storeLock,
  80. storeCache,
  81. validUntil
  82. };
  83. };
  84. /**
  85. * @typedef {Object} LockfileEntry
  86. * @property {string} resolved
  87. * @property {string} integrity
  88. * @property {string} contentType
  89. */
  90. const areLockfileEntriesEqual = (a, b) => {
  91. return (
  92. a.resolved === b.resolved &&
  93. a.integrity === b.integrity &&
  94. a.contentType === b.contentType
  95. );
  96. };
  97. const entryToString = entry => {
  98. return `resolved: ${entry.resolved}, integrity: ${entry.integrity}, contentType: ${entry.contentType}`;
  99. };
  100. class Lockfile {
  101. constructor() {
  102. this.version = 1;
  103. /** @type {Map<string, LockfileEntry | "ignore" | "no-cache">} */
  104. this.entries = new Map();
  105. }
  106. static parse(content) {
  107. // TODO handle merge conflicts
  108. const data = JSON.parse(content);
  109. if (data.version !== 1)
  110. throw new Error(`Unsupported lockfile version ${data.version}`);
  111. const lockfile = new Lockfile();
  112. for (const key of Object.keys(data)) {
  113. if (key === "version") continue;
  114. const entry = data[key];
  115. lockfile.entries.set(
  116. key,
  117. typeof entry === "string"
  118. ? entry
  119. : {
  120. resolved: key,
  121. ...entry
  122. }
  123. );
  124. }
  125. return lockfile;
  126. }
  127. toString() {
  128. let str = "{\n";
  129. const entries = Array.from(this.entries).sort(([a], [b]) =>
  130. a < b ? -1 : 1
  131. );
  132. for (const [key, entry] of entries) {
  133. if (typeof entry === "string") {
  134. str += ` ${JSON.stringify(key)}: ${JSON.stringify(entry)},\n`;
  135. } else {
  136. str += ` ${JSON.stringify(key)}: { `;
  137. if (entry.resolved !== key)
  138. str += `"resolved": ${JSON.stringify(entry.resolved)}, `;
  139. str += `"integrity": ${JSON.stringify(
  140. entry.integrity
  141. )}, "contentType": ${JSON.stringify(entry.contentType)} },\n`;
  142. }
  143. }
  144. str += ` "version": ${this.version}\n}\n`;
  145. return str;
  146. }
  147. }
  148. /**
  149. * @template R
  150. * @param {function(function(Error=, R=): void): void} fn function
  151. * @returns {function(function((Error | null)=, R=): void): void} cached function
  152. */
  153. const cachedWithoutKey = fn => {
  154. let inFlight = false;
  155. /** @type {Error | undefined} */
  156. let cachedError = undefined;
  157. /** @type {R | undefined} */
  158. let cachedResult = undefined;
  159. /** @type {(function(Error=, R=): void)[] | undefined} */
  160. let cachedCallbacks = undefined;
  161. return callback => {
  162. if (inFlight) {
  163. if (cachedResult !== undefined) return callback(null, cachedResult);
  164. if (cachedError !== undefined) return callback(cachedError);
  165. if (cachedCallbacks === undefined) cachedCallbacks = [callback];
  166. else cachedCallbacks.push(callback);
  167. return;
  168. }
  169. inFlight = true;
  170. fn((err, result) => {
  171. if (err) cachedError = err;
  172. else cachedResult = result;
  173. const callbacks = cachedCallbacks;
  174. cachedCallbacks = undefined;
  175. callback(err, result);
  176. if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
  177. });
  178. };
  179. };
  180. /**
  181. * @template T
  182. * @template R
  183. * @param {function(T, function(Error=, R=): void): void} fn function
  184. * @param {function(T, function(Error=, R=): void): void=} forceFn function for the second try
  185. * @returns {(function(T, function((Error | null)=, R=): void): void) & { force: function(T, function((Error | null)=, R=): void): void }} cached function
  186. */
  187. const cachedWithKey = (fn, forceFn = fn) => {
  188. /** @typedef {{ result?: R, error?: Error, callbacks?: (function((Error | null)=, R=): void)[], force?: true }} CacheEntry */
  189. /** @type {Map<T, CacheEntry>} */
  190. const cache = new Map();
  191. const resultFn = (arg, callback) => {
  192. const cacheEntry = cache.get(arg);
  193. if (cacheEntry !== undefined) {
  194. if (cacheEntry.result !== undefined)
  195. return callback(null, cacheEntry.result);
  196. if (cacheEntry.error !== undefined) return callback(cacheEntry.error);
  197. if (cacheEntry.callbacks === undefined) cacheEntry.callbacks = [callback];
  198. else cacheEntry.callbacks.push(callback);
  199. return;
  200. }
  201. /** @type {CacheEntry} */
  202. const newCacheEntry = {
  203. result: undefined,
  204. error: undefined,
  205. callbacks: undefined
  206. };
  207. cache.set(arg, newCacheEntry);
  208. fn(arg, (err, result) => {
  209. if (err) newCacheEntry.error = err;
  210. else newCacheEntry.result = result;
  211. const callbacks = newCacheEntry.callbacks;
  212. newCacheEntry.callbacks = undefined;
  213. callback(err, result);
  214. if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
  215. });
  216. };
  217. resultFn.force = (arg, callback) => {
  218. const cacheEntry = cache.get(arg);
  219. if (cacheEntry !== undefined && cacheEntry.force) {
  220. if (cacheEntry.result !== undefined)
  221. return callback(null, cacheEntry.result);
  222. if (cacheEntry.error !== undefined) return callback(cacheEntry.error);
  223. if (cacheEntry.callbacks === undefined) cacheEntry.callbacks = [callback];
  224. else cacheEntry.callbacks.push(callback);
  225. return;
  226. }
  227. /** @type {CacheEntry} */
  228. const newCacheEntry = {
  229. result: undefined,
  230. error: undefined,
  231. callbacks: undefined,
  232. force: true
  233. };
  234. cache.set(arg, newCacheEntry);
  235. forceFn(arg, (err, result) => {
  236. if (err) newCacheEntry.error = err;
  237. else newCacheEntry.result = result;
  238. const callbacks = newCacheEntry.callbacks;
  239. newCacheEntry.callbacks = undefined;
  240. callback(err, result);
  241. if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
  242. });
  243. };
  244. return resultFn;
  245. };
  246. class HttpUriPlugin {
  247. /**
  248. * @param {HttpUriPluginOptions} options options
  249. */
  250. constructor(options) {
  251. validate(options);
  252. this._lockfileLocation = options.lockfileLocation;
  253. this._cacheLocation = options.cacheLocation;
  254. this._upgrade = options.upgrade;
  255. this._frozen = options.frozen;
  256. this._allowedUris = options.allowedUris;
  257. }
  258. /**
  259. * Apply the plugin
  260. * @param {Compiler} compiler the compiler instance
  261. * @returns {void}
  262. */
  263. apply(compiler) {
  264. const schemes = [
  265. {
  266. scheme: "http",
  267. fetch: (url, options, callback) => getHttp().get(url, options, callback)
  268. },
  269. {
  270. scheme: "https",
  271. fetch: (url, options, callback) =>
  272. getHttps().get(url, options, callback)
  273. }
  274. ];
  275. let lockfileCache;
  276. compiler.hooks.compilation.tap(
  277. "HttpUriPlugin",
  278. (compilation, { normalModuleFactory }) => {
  279. const intermediateFs = compiler.intermediateFileSystem;
  280. const fs = compilation.inputFileSystem;
  281. const cache = compilation.getCache("webpack.HttpUriPlugin");
  282. const logger = compilation.getLogger("webpack.HttpUriPlugin");
  283. const lockfileLocation =
  284. this._lockfileLocation ||
  285. join(
  286. intermediateFs,
  287. compiler.context,
  288. compiler.name
  289. ? `${toSafePath(compiler.name)}.webpack.lock`
  290. : "webpack.lock"
  291. );
  292. const cacheLocation =
  293. this._cacheLocation !== undefined
  294. ? this._cacheLocation
  295. : lockfileLocation + ".data";
  296. const upgrade = this._upgrade || false;
  297. const frozen = this._frozen || false;
  298. const hashFunction = "sha512";
  299. const hashDigest = "hex";
  300. const hashDigestLength = 20;
  301. const allowedUris = this._allowedUris;
  302. let warnedAboutEol = false;
  303. const cacheKeyCache = new Map();
  304. /**
  305. * @param {string} url the url
  306. * @returns {string} the key
  307. */
  308. const getCacheKey = url => {
  309. const cachedResult = cacheKeyCache.get(url);
  310. if (cachedResult !== undefined) return cachedResult;
  311. const result = _getCacheKey(url);
  312. cacheKeyCache.set(url, result);
  313. return result;
  314. };
  315. /**
  316. * @param {string} url the url
  317. * @returns {string} the key
  318. */
  319. const _getCacheKey = url => {
  320. const parsedUrl = new URL(url);
  321. const folder = toSafePath(parsedUrl.origin);
  322. const name = toSafePath(parsedUrl.pathname);
  323. const query = toSafePath(parsedUrl.search);
  324. let ext = extname(name);
  325. if (ext.length > 20) ext = "";
  326. const basename = ext ? name.slice(0, -ext.length) : name;
  327. const hash = createHash(hashFunction);
  328. hash.update(url);
  329. const digest = hash.digest(hashDigest).slice(0, hashDigestLength);
  330. return `${folder.slice(-50)}/${`${basename}${
  331. query ? `_${query}` : ""
  332. }`.slice(0, 150)}_${digest}${ext}`;
  333. };
  334. const getLockfile = cachedWithoutKey(
  335. /**
  336. * @param {function((Error | null)=, Lockfile=): void} callback callback
  337. * @returns {void}
  338. */
  339. callback => {
  340. const readLockfile = () => {
  341. intermediateFs.readFile(lockfileLocation, (err, buffer) => {
  342. if (err && err.code !== "ENOENT") {
  343. compilation.missingDependencies.add(lockfileLocation);
  344. return callback(err);
  345. }
  346. compilation.fileDependencies.add(lockfileLocation);
  347. compilation.fileSystemInfo.createSnapshot(
  348. compiler.fsStartTime,
  349. buffer ? [lockfileLocation] : [],
  350. [],
  351. buffer ? [] : [lockfileLocation],
  352. { timestamp: true },
  353. (err, snapshot) => {
  354. if (err) return callback(err);
  355. const lockfile = buffer
  356. ? Lockfile.parse(buffer.toString("utf-8"))
  357. : new Lockfile();
  358. lockfileCache = {
  359. lockfile,
  360. snapshot
  361. };
  362. callback(null, lockfile);
  363. }
  364. );
  365. });
  366. };
  367. if (lockfileCache) {
  368. compilation.fileSystemInfo.checkSnapshotValid(
  369. lockfileCache.snapshot,
  370. (err, valid) => {
  371. if (err) return callback(err);
  372. if (!valid) return readLockfile();
  373. callback(null, lockfileCache.lockfile);
  374. }
  375. );
  376. } else {
  377. readLockfile();
  378. }
  379. }
  380. );
  381. /** @type {Map<string, LockfileEntry | "ignore" | "no-cache"> | undefined} */
  382. let lockfileUpdates = undefined;
  383. const storeLockEntry = (lockfile, url, entry) => {
  384. const oldEntry = lockfile.entries.get(url);
  385. if (lockfileUpdates === undefined) lockfileUpdates = new Map();
  386. lockfileUpdates.set(url, entry);
  387. lockfile.entries.set(url, entry);
  388. if (!oldEntry) {
  389. logger.log(`${url} added to lockfile`);
  390. } else if (typeof oldEntry === "string") {
  391. if (typeof entry === "string") {
  392. logger.log(`${url} updated in lockfile: ${oldEntry} -> ${entry}`);
  393. } else {
  394. logger.log(
  395. `${url} updated in lockfile: ${oldEntry} -> ${entry.resolved}`
  396. );
  397. }
  398. } else if (typeof entry === "string") {
  399. logger.log(
  400. `${url} updated in lockfile: ${oldEntry.resolved} -> ${entry}`
  401. );
  402. } else if (oldEntry.resolved !== entry.resolved) {
  403. logger.log(
  404. `${url} updated in lockfile: ${oldEntry.resolved} -> ${entry.resolved}`
  405. );
  406. } else if (oldEntry.integrity !== entry.integrity) {
  407. logger.log(`${url} updated in lockfile: content changed`);
  408. } else if (oldEntry.contentType !== entry.contentType) {
  409. logger.log(
  410. `${url} updated in lockfile: ${oldEntry.contentType} -> ${entry.contentType}`
  411. );
  412. } else {
  413. logger.log(`${url} updated in lockfile`);
  414. }
  415. };
  416. const storeResult = (lockfile, url, result, callback) => {
  417. if (result.storeLock) {
  418. storeLockEntry(lockfile, url, result.entry);
  419. if (!cacheLocation || !result.content)
  420. return callback(null, result);
  421. const key = getCacheKey(result.entry.resolved);
  422. const filePath = join(intermediateFs, cacheLocation, key);
  423. mkdirp(intermediateFs, dirname(intermediateFs, filePath), err => {
  424. if (err) return callback(err);
  425. intermediateFs.writeFile(filePath, result.content, err => {
  426. if (err) return callback(err);
  427. callback(null, result);
  428. });
  429. });
  430. } else {
  431. storeLockEntry(lockfile, url, "no-cache");
  432. callback(null, result);
  433. }
  434. };
  435. for (const { scheme, fetch } of schemes) {
  436. /**
  437. *
  438. * @param {string} url URL
  439. * @param {string} integrity integrity
  440. * @param {function((Error | null)=, { entry: LockfileEntry, content: Buffer, storeLock: boolean }=): void} callback callback
  441. */
  442. const resolveContent = (url, integrity, callback) => {
  443. const handleResult = (err, result) => {
  444. if (err) return callback(err);
  445. if ("location" in result) {
  446. return resolveContent(
  447. result.location,
  448. integrity,
  449. (err, innerResult) => {
  450. if (err) return callback(err);
  451. callback(null, {
  452. entry: innerResult.entry,
  453. content: innerResult.content,
  454. storeLock: innerResult.storeLock && result.storeLock
  455. });
  456. }
  457. );
  458. } else {
  459. if (
  460. !result.fresh &&
  461. integrity &&
  462. result.entry.integrity !== integrity &&
  463. !verifyIntegrity(result.content, integrity)
  464. ) {
  465. return fetchContent.force(url, handleResult);
  466. }
  467. return callback(null, {
  468. entry: result.entry,
  469. content: result.content,
  470. storeLock: result.storeLock
  471. });
  472. }
  473. };
  474. fetchContent(url, handleResult);
  475. };
  476. /** @typedef {{ storeCache: boolean, storeLock: boolean, validUntil: number, etag: string | undefined, fresh: boolean }} FetchResultMeta */
  477. /** @typedef {FetchResultMeta & { location: string }} RedirectFetchResult */
  478. /** @typedef {FetchResultMeta & { entry: LockfileEntry, content: Buffer }} ContentFetchResult */
  479. /** @typedef {RedirectFetchResult | ContentFetchResult} FetchResult */
  480. /**
  481. * @param {string} url URL
  482. * @param {FetchResult | RedirectFetchResult} cachedResult result from cache
  483. * @param {function((Error | null)=, FetchResult=): void} callback callback
  484. * @returns {void}
  485. */
  486. const fetchContentRaw = (url, cachedResult, callback) => {
  487. const requestTime = Date.now();
  488. fetch(
  489. new URL(url),
  490. {
  491. headers: {
  492. "accept-encoding": "gzip, deflate, br",
  493. "user-agent": "webpack",
  494. "if-none-match": cachedResult
  495. ? cachedResult.etag || null
  496. : null
  497. }
  498. },
  499. res => {
  500. const etag = res.headers["etag"];
  501. const location = res.headers["location"];
  502. const cacheControl = res.headers["cache-control"];
  503. const { storeLock, storeCache, validUntil } = parseCacheControl(
  504. cacheControl,
  505. requestTime
  506. );
  507. /**
  508. * @param {Partial<Pick<FetchResultMeta, "fresh">> & (Pick<RedirectFetchResult, "location"> | Pick<ContentFetchResult, "content" | "entry">)} partialResult result
  509. * @returns {void}
  510. */
  511. const finishWith = partialResult => {
  512. if ("location" in partialResult) {
  513. logger.debug(
  514. `GET ${url} [${res.statusCode}] -> ${partialResult.location}`
  515. );
  516. } else {
  517. logger.debug(
  518. `GET ${url} [${res.statusCode}] ${Math.ceil(
  519. partialResult.content.length / 1024
  520. )} kB${!storeLock ? " no-cache" : ""}`
  521. );
  522. }
  523. const result = {
  524. ...partialResult,
  525. fresh: true,
  526. storeLock,
  527. storeCache,
  528. validUntil,
  529. etag
  530. };
  531. if (!storeCache) {
  532. logger.log(
  533. `${url} can't be stored in cache, due to Cache-Control header: ${cacheControl}`
  534. );
  535. return callback(null, result);
  536. }
  537. cache.store(
  538. url,
  539. null,
  540. {
  541. ...result,
  542. fresh: false
  543. },
  544. err => {
  545. if (err) {
  546. logger.warn(
  547. `${url} can't be stored in cache: ${err.message}`
  548. );
  549. logger.debug(err.stack);
  550. }
  551. callback(null, result);
  552. }
  553. );
  554. };
  555. if (res.statusCode === 304) {
  556. if (
  557. cachedResult.validUntil < validUntil ||
  558. cachedResult.storeLock !== storeLock ||
  559. cachedResult.storeCache !== storeCache ||
  560. cachedResult.etag !== etag
  561. ) {
  562. return finishWith(cachedResult);
  563. } else {
  564. logger.debug(`GET ${url} [${res.statusCode}] (unchanged)`);
  565. return callback(null, {
  566. ...cachedResult,
  567. fresh: true
  568. });
  569. }
  570. }
  571. if (
  572. location &&
  573. res.statusCode >= 301 &&
  574. res.statusCode <= 308
  575. ) {
  576. const result = {
  577. location: new URL(location, url).href
  578. };
  579. if (
  580. !cachedResult ||
  581. !("location" in cachedResult) ||
  582. cachedResult.location !== result.location ||
  583. cachedResult.validUntil < validUntil ||
  584. cachedResult.storeLock !== storeLock ||
  585. cachedResult.storeCache !== storeCache ||
  586. cachedResult.etag !== etag
  587. ) {
  588. return finishWith(result);
  589. } else {
  590. logger.debug(`GET ${url} [${res.statusCode}] (unchanged)`);
  591. return callback(null, {
  592. ...result,
  593. fresh: true,
  594. storeLock,
  595. storeCache,
  596. validUntil,
  597. etag
  598. });
  599. }
  600. }
  601. const contentType = res.headers["content-type"] || "";
  602. const bufferArr = [];
  603. const contentEncoding = res.headers["content-encoding"];
  604. let stream = res;
  605. if (contentEncoding === "gzip") {
  606. stream = stream.pipe(createGunzip());
  607. } else if (contentEncoding === "br") {
  608. stream = stream.pipe(createBrotliDecompress());
  609. } else if (contentEncoding === "deflate") {
  610. stream = stream.pipe(createInflate());
  611. }
  612. stream.on("data", chunk => {
  613. bufferArr.push(chunk);
  614. });
  615. stream.on("end", () => {
  616. if (!res.complete) {
  617. logger.log(`GET ${url} [${res.statusCode}] (terminated)`);
  618. return callback(new Error(`${url} request was terminated`));
  619. }
  620. const content = Buffer.concat(bufferArr);
  621. if (res.statusCode !== 200) {
  622. logger.log(`GET ${url} [${res.statusCode}]`);
  623. return callback(
  624. new Error(
  625. `${url} request status code = ${
  626. res.statusCode
  627. }\n${content.toString("utf-8")}`
  628. )
  629. );
  630. }
  631. const integrity = computeIntegrity(content);
  632. const entry = { resolved: url, integrity, contentType };
  633. finishWith({
  634. entry,
  635. content
  636. });
  637. });
  638. }
  639. ).on("error", err => {
  640. logger.log(`GET ${url} (error)`);
  641. err.message += `\nwhile fetching ${url}`;
  642. callback(err);
  643. });
  644. };
  645. const fetchContent = cachedWithKey(
  646. /**
  647. * @param {string} url URL
  648. * @param {function((Error | null)=, { validUntil: number, etag?: string, entry: LockfileEntry, content: Buffer, fresh: boolean } | { validUntil: number, etag?: string, location: string, fresh: boolean }=): void} callback callback
  649. * @returns {void}
  650. */ (url, callback) => {
  651. cache.get(url, null, (err, cachedResult) => {
  652. if (err) return callback(err);
  653. if (cachedResult) {
  654. const isValid = cachedResult.validUntil >= Date.now();
  655. if (isValid) return callback(null, cachedResult);
  656. }
  657. fetchContentRaw(url, cachedResult, callback);
  658. });
  659. },
  660. (url, callback) => fetchContentRaw(url, undefined, callback)
  661. );
  662. const isAllowed = uri => {
  663. for (const allowed of allowedUris) {
  664. if (typeof allowed === "string") {
  665. if (uri.startsWith(allowed)) return true;
  666. } else if (typeof allowed === "function") {
  667. if (allowed(uri)) return true;
  668. } else {
  669. if (allowed.test(uri)) return true;
  670. }
  671. }
  672. return false;
  673. };
  674. const getInfo = cachedWithKey(
  675. /**
  676. * @param {string} url the url
  677. * @param {function((Error | null)=, { entry: LockfileEntry, content: Buffer }=): void} callback callback
  678. * @returns {void}
  679. */
  680. (url, callback) => {
  681. if (!isAllowed(url)) {
  682. return callback(
  683. new Error(
  684. `${url} doesn't match the allowedUris policy. These URIs are allowed:\n${allowedUris
  685. .map(uri => ` - ${uri}`)
  686. .join("\n")}`
  687. )
  688. );
  689. }
  690. getLockfile((err, lockfile) => {
  691. if (err) return callback(err);
  692. const entryOrString = lockfile.entries.get(url);
  693. if (!entryOrString) {
  694. if (frozen) {
  695. return callback(
  696. new Error(
  697. `${url} has no lockfile entry and lockfile is frozen`
  698. )
  699. );
  700. }
  701. resolveContent(url, null, (err, result) => {
  702. if (err) return callback(err);
  703. storeResult(lockfile, url, result, callback);
  704. });
  705. return;
  706. }
  707. if (typeof entryOrString === "string") {
  708. const entryTag = entryOrString;
  709. resolveContent(url, null, (err, result) => {
  710. if (err) return callback(err);
  711. if (!result.storeLock || entryTag === "ignore")
  712. return callback(null, result);
  713. if (frozen) {
  714. return callback(
  715. new Error(
  716. `${url} used to have ${entryTag} lockfile entry and has content now, but lockfile is frozen`
  717. )
  718. );
  719. }
  720. if (!upgrade) {
  721. return callback(
  722. new Error(
  723. `${url} used to have ${entryTag} lockfile entry and has content now.
  724. This should be reflected in the lockfile, so this lockfile entry must be upgraded, but upgrading is not enabled.
  725. Remove this line from the lockfile to force upgrading.`
  726. )
  727. );
  728. }
  729. storeResult(lockfile, url, result, callback);
  730. });
  731. return;
  732. }
  733. let entry = entryOrString;
  734. const doFetch = lockedContent => {
  735. resolveContent(url, entry.integrity, (err, result) => {
  736. if (err) {
  737. if (lockedContent) {
  738. logger.warn(
  739. `Upgrade request to ${url} failed: ${err.message}`
  740. );
  741. logger.debug(err.stack);
  742. return callback(null, {
  743. entry,
  744. content: lockedContent
  745. });
  746. }
  747. return callback(err);
  748. }
  749. if (!result.storeLock) {
  750. // When the lockfile entry should be no-cache
  751. // we need to update the lockfile
  752. if (frozen) {
  753. return callback(
  754. new Error(
  755. `${url} has a lockfile entry and is no-cache now, but lockfile is frozen\nLockfile: ${entryToString(
  756. entry
  757. )}`
  758. )
  759. );
  760. }
  761. storeResult(lockfile, url, result, callback);
  762. return;
  763. }
  764. if (!areLockfileEntriesEqual(result.entry, entry)) {
  765. // When the lockfile entry is outdated
  766. // we need to update the lockfile
  767. if (frozen) {
  768. return callback(
  769. new Error(
  770. `${url} has an outdated lockfile entry, but lockfile is frozen\nLockfile: ${entryToString(
  771. entry
  772. )}\nExpected: ${entryToString(result.entry)}`
  773. )
  774. );
  775. }
  776. storeResult(lockfile, url, result, callback);
  777. return;
  778. }
  779. if (!lockedContent && cacheLocation) {
  780. // When the lockfile cache content is missing
  781. // we need to update the lockfile
  782. if (frozen) {
  783. return callback(
  784. new Error(
  785. `${url} is missing content in the lockfile cache, but lockfile is frozen\nLockfile: ${entryToString(
  786. entry
  787. )}`
  788. )
  789. );
  790. }
  791. storeResult(lockfile, url, result, callback);
  792. return;
  793. }
  794. return callback(null, result);
  795. });
  796. };
  797. if (cacheLocation) {
  798. // When there is a lockfile cache
  799. // we read the content from there
  800. const key = getCacheKey(entry.resolved);
  801. const filePath = join(intermediateFs, cacheLocation, key);
  802. fs.readFile(filePath, (err, result) => {
  803. const content = /** @type {Buffer} */ (result);
  804. if (err) {
  805. if (err.code === "ENOENT") return doFetch();
  806. return callback(err);
  807. }
  808. const continueWithCachedContent = result => {
  809. if (!upgrade) {
  810. // When not in upgrade mode, we accept the result from the lockfile cache
  811. return callback(null, { entry, content });
  812. }
  813. return doFetch(content);
  814. };
  815. if (!verifyIntegrity(content, entry.integrity)) {
  816. let contentWithChangedEol;
  817. let isEolChanged = false;
  818. try {
  819. contentWithChangedEol = Buffer.from(
  820. content.toString("utf-8").replace(/\r\n/g, "\n")
  821. );
  822. isEolChanged = verifyIntegrity(
  823. contentWithChangedEol,
  824. entry.integrity
  825. );
  826. } catch (e) {
  827. // ignore
  828. }
  829. if (isEolChanged) {
  830. if (!warnedAboutEol) {
  831. const explainer = `Incorrect end of line sequence was detected in the lockfile cache.
  832. The lockfile cache is protected by integrity checks, so any external modification will lead to a corrupted lockfile cache.
  833. When using git make sure to configure .gitattributes correctly for the lockfile cache:
  834. **/*webpack.lock.data/** -text
  835. This will avoid that the end of line sequence is changed by git on Windows.`;
  836. if (frozen) {
  837. logger.error(explainer);
  838. } else {
  839. logger.warn(explainer);
  840. logger.info(
  841. "Lockfile cache will be automatically fixed now, but when lockfile is frozen this would result in an error."
  842. );
  843. }
  844. warnedAboutEol = true;
  845. }
  846. if (!frozen) {
  847. // "fix" the end of line sequence of the lockfile content
  848. logger.log(
  849. `${filePath} fixed end of line sequence (\\r\\n instead of \\n).`
  850. );
  851. intermediateFs.writeFile(
  852. filePath,
  853. contentWithChangedEol,
  854. err => {
  855. if (err) return callback(err);
  856. continueWithCachedContent(contentWithChangedEol);
  857. }
  858. );
  859. return;
  860. }
  861. }
  862. if (frozen) {
  863. return callback(
  864. new Error(
  865. `${
  866. entry.resolved
  867. } integrity mismatch, expected content with integrity ${
  868. entry.integrity
  869. } but got ${computeIntegrity(content)}.
  870. Lockfile corrupted (${
  871. isEolChanged
  872. ? "end of line sequence was unexpectedly changed"
  873. : "incorrectly merged? changed by other tools?"
  874. }).
  875. Run build with un-frozen lockfile to automatically fix lockfile.`
  876. )
  877. );
  878. } else {
  879. // "fix" the lockfile entry to the correct integrity
  880. // the content has priority over the integrity value
  881. entry = {
  882. ...entry,
  883. integrity: computeIntegrity(content)
  884. };
  885. storeLockEntry(lockfile, url, entry);
  886. }
  887. }
  888. continueWithCachedContent(result);
  889. });
  890. } else {
  891. doFetch();
  892. }
  893. });
  894. }
  895. );
  896. const respondWithUrlModule = (url, resourceData, callback) => {
  897. getInfo(url.href, (err, result) => {
  898. if (err) return callback(err);
  899. resourceData.resource = url.href;
  900. resourceData.path = url.origin + url.pathname;
  901. resourceData.query = url.search;
  902. resourceData.fragment = url.hash;
  903. resourceData.context = new URL(
  904. ".",
  905. result.entry.resolved
  906. ).href.slice(0, -1);
  907. resourceData.data.mimetype = result.entry.contentType;
  908. callback(null, true);
  909. });
  910. };
  911. normalModuleFactory.hooks.resolveForScheme
  912. .for(scheme)
  913. .tapAsync(
  914. "HttpUriPlugin",
  915. (resourceData, resolveData, callback) => {
  916. respondWithUrlModule(
  917. new URL(resourceData.resource),
  918. resourceData,
  919. callback
  920. );
  921. }
  922. );
  923. normalModuleFactory.hooks.resolveInScheme
  924. .for(scheme)
  925. .tapAsync("HttpUriPlugin", (resourceData, data, callback) => {
  926. // Only handle relative urls (./xxx, ../xxx, /xxx, //xxx)
  927. if (
  928. data.dependencyType !== "url" &&
  929. !/^\.{0,2}\//.test(resourceData.resource)
  930. ) {
  931. return callback();
  932. }
  933. respondWithUrlModule(
  934. new URL(resourceData.resource, data.context + "/"),
  935. resourceData,
  936. callback
  937. );
  938. });
  939. const hooks = NormalModule.getCompilationHooks(compilation);
  940. hooks.readResourceForScheme
  941. .for(scheme)
  942. .tapAsync("HttpUriPlugin", (resource, module, callback) => {
  943. return getInfo(resource, (err, result) => {
  944. if (err) return callback(err);
  945. module.buildInfo.resourceIntegrity = result.entry.integrity;
  946. callback(null, result.content);
  947. });
  948. });
  949. hooks.needBuild.tapAsync(
  950. "HttpUriPlugin",
  951. (module, context, callback) => {
  952. if (
  953. module.resource &&
  954. module.resource.startsWith(`${scheme}://`)
  955. ) {
  956. getInfo(module.resource, (err, result) => {
  957. if (err) return callback(err);
  958. if (
  959. result.entry.integrity !==
  960. module.buildInfo.resourceIntegrity
  961. ) {
  962. return callback(null, true);
  963. }
  964. callback();
  965. });
  966. } else {
  967. return callback();
  968. }
  969. }
  970. );
  971. }
  972. compilation.hooks.finishModules.tapAsync(
  973. "HttpUriPlugin",
  974. (modules, callback) => {
  975. if (!lockfileUpdates) return callback();
  976. const ext = extname(lockfileLocation);
  977. const tempFile = join(
  978. intermediateFs,
  979. dirname(intermediateFs, lockfileLocation),
  980. `.${basename(lockfileLocation, ext)}.${
  981. (Math.random() * 10000) | 0
  982. }${ext}`
  983. );
  984. const writeDone = () => {
  985. const nextOperation = inProgressWrite.shift();
  986. if (nextOperation) {
  987. nextOperation();
  988. } else {
  989. inProgressWrite = undefined;
  990. }
  991. };
  992. const runWrite = () => {
  993. intermediateFs.readFile(lockfileLocation, (err, buffer) => {
  994. if (err && err.code !== "ENOENT") {
  995. writeDone();
  996. return callback(err);
  997. }
  998. const lockfile = buffer
  999. ? Lockfile.parse(buffer.toString("utf-8"))
  1000. : new Lockfile();
  1001. for (const [key, value] of lockfileUpdates) {
  1002. lockfile.entries.set(key, value);
  1003. }
  1004. intermediateFs.writeFile(tempFile, lockfile.toString(), err => {
  1005. if (err) {
  1006. writeDone();
  1007. return intermediateFs.unlink(tempFile, () => callback(err));
  1008. }
  1009. intermediateFs.rename(tempFile, lockfileLocation, err => {
  1010. if (err) {
  1011. writeDone();
  1012. return intermediateFs.unlink(tempFile, () =>
  1013. callback(err)
  1014. );
  1015. }
  1016. writeDone();
  1017. callback();
  1018. });
  1019. });
  1020. });
  1021. };
  1022. if (inProgressWrite) {
  1023. inProgressWrite.push(runWrite);
  1024. } else {
  1025. inProgressWrite = [];
  1026. runWrite();
  1027. }
  1028. }
  1029. );
  1030. }
  1031. );
  1032. }
  1033. }
  1034. module.exports = HttpUriPlugin;