mirror of
https://github.com/titanscouting/tra-analysis.git
synced 2025-03-15 18:15:46 +00:00
483 lines
14 KiB
JavaScript
483 lines
14 KiB
JavaScript
'use strict'
|
|
|
|
let Cache
|
|
const url = require('url')
|
|
const CachePolicy = require('http-cache-semantics')
|
|
const fetch = require('node-fetch-npm')
|
|
const pkg = require('./package.json')
|
|
const retry = require('promise-retry')
|
|
let ssri
|
|
const Stream = require('stream')
|
|
const getAgent = require('./agent')
|
|
const setWarning = require('./warning')
|
|
|
|
const isURL = /^https?:/
|
|
const USER_AGENT = `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})`
|
|
|
|
const RETRY_ERRORS = [
|
|
'ECONNRESET', // remote socket closed on us
|
|
'ECONNREFUSED', // remote host refused to open connection
|
|
'EADDRINUSE', // failed to bind to a local port (proxy?)
|
|
'ETIMEDOUT' // someone in the transaction is WAY TOO SLOW
|
|
// Known codes we do NOT retry on:
|
|
// ENOTFOUND (getaddrinfo failure. Either bad hostname, or offline)
|
|
]
|
|
|
|
const RETRY_TYPES = [
|
|
'request-timeout'
|
|
]
|
|
|
|
// https://fetch.spec.whatwg.org/#http-network-or-cache-fetch
|
|
module.exports = cachingFetch
|
|
cachingFetch.defaults = function (_uri, _opts) {
|
|
const fetch = this
|
|
if (typeof _uri === 'object') {
|
|
_opts = _uri
|
|
_uri = null
|
|
}
|
|
|
|
function defaultedFetch (uri, opts) {
|
|
const finalOpts = Object.assign({}, _opts || {}, opts || {})
|
|
return fetch(uri || _uri, finalOpts)
|
|
}
|
|
|
|
defaultedFetch.defaults = fetch.defaults
|
|
defaultedFetch.delete = fetch.delete
|
|
return defaultedFetch
|
|
}
|
|
|
|
cachingFetch.delete = cacheDelete
|
|
function cacheDelete (uri, opts) {
|
|
opts = configureOptions(opts)
|
|
if (opts.cacheManager) {
|
|
const req = new fetch.Request(uri, {
|
|
method: opts.method,
|
|
headers: opts.headers
|
|
})
|
|
return opts.cacheManager.delete(req, opts)
|
|
}
|
|
}
|
|
|
|
function initializeCache (opts) {
|
|
if (typeof opts.cacheManager === 'string') {
|
|
if (!Cache) {
|
|
// Default cacache-based cache
|
|
Cache = require('./cache')
|
|
}
|
|
|
|
opts.cacheManager = new Cache(opts.cacheManager, opts)
|
|
}
|
|
|
|
opts.cache = opts.cache || 'default'
|
|
|
|
if (opts.cache === 'default' && isHeaderConditional(opts.headers)) {
|
|
// If header list contains `If-Modified-Since`, `If-None-Match`,
|
|
// `If-Unmodified-Since`, `If-Match`, or `If-Range`, fetch will set cache
|
|
// mode to "no-store" if it is "default".
|
|
opts.cache = 'no-store'
|
|
}
|
|
}
|
|
|
|
function configureOptions (_opts) {
|
|
const opts = Object.assign({}, _opts || {})
|
|
opts.method = (opts.method || 'GET').toUpperCase()
|
|
|
|
if (opts.retry && typeof opts.retry === 'number') {
|
|
opts.retry = { retries: opts.retry }
|
|
}
|
|
|
|
if (opts.retry === false) {
|
|
opts.retry = { retries: 0 }
|
|
}
|
|
|
|
if (opts.cacheManager) {
|
|
initializeCache(opts)
|
|
}
|
|
|
|
return opts
|
|
}
|
|
|
|
function initializeSsri () {
|
|
if (!ssri) {
|
|
ssri = require('ssri')
|
|
}
|
|
}
|
|
|
|
function cachingFetch (uri, _opts) {
|
|
const opts = configureOptions(_opts)
|
|
|
|
if (opts.integrity) {
|
|
initializeSsri()
|
|
// if verifying integrity, node-fetch must not decompress
|
|
opts.compress = false
|
|
}
|
|
|
|
const isCachable = (opts.method === 'GET' || opts.method === 'HEAD') &&
|
|
opts.cacheManager &&
|
|
opts.cache !== 'no-store' &&
|
|
opts.cache !== 'reload'
|
|
|
|
if (isCachable) {
|
|
const req = new fetch.Request(uri, {
|
|
method: opts.method,
|
|
headers: opts.headers
|
|
})
|
|
|
|
return opts.cacheManager.match(req, opts).then(res => {
|
|
if (res) {
|
|
const warningCode = (res.headers.get('Warning') || '').match(/^\d+/)
|
|
if (warningCode && +warningCode >= 100 && +warningCode < 200) {
|
|
// https://tools.ietf.org/html/rfc7234#section-4.3.4
|
|
//
|
|
// If a stored response is selected for update, the cache MUST:
|
|
//
|
|
// * delete any Warning header fields in the stored response with
|
|
// warn-code 1xx (see Section 5.5);
|
|
//
|
|
// * retain any Warning header fields in the stored response with
|
|
// warn-code 2xx;
|
|
//
|
|
res.headers.delete('Warning')
|
|
}
|
|
|
|
if (opts.cache === 'default' && !isStale(req, res)) {
|
|
return res
|
|
}
|
|
|
|
if (opts.cache === 'default' || opts.cache === 'no-cache') {
|
|
return conditionalFetch(req, res, opts)
|
|
}
|
|
|
|
if (opts.cache === 'force-cache' || opts.cache === 'only-if-cached') {
|
|
// 112 Disconnected operation
|
|
// SHOULD be included if the cache is intentionally disconnected from
|
|
// the rest of the network for a period of time.
|
|
// (https://tools.ietf.org/html/rfc2616#section-14.46)
|
|
setWarning(res, 112, 'Disconnected operation')
|
|
return res
|
|
}
|
|
}
|
|
|
|
if (!res && opts.cache === 'only-if-cached') {
|
|
const errorMsg = `request to ${
|
|
uri
|
|
} failed: cache mode is 'only-if-cached' but no cached response available.`
|
|
|
|
const err = new Error(errorMsg)
|
|
err.code = 'ENOTCACHED'
|
|
throw err
|
|
}
|
|
|
|
// Missing cache entry, or mode is default (if stale), reload, no-store
|
|
return remoteFetch(req.url, opts)
|
|
})
|
|
}
|
|
|
|
return remoteFetch(uri, opts)
|
|
}
|
|
|
|
function iterableToObject (iter) {
|
|
const obj = {}
|
|
for (let k of iter.keys()) {
|
|
obj[k] = iter.get(k)
|
|
}
|
|
return obj
|
|
}
|
|
|
|
function makePolicy (req, res) {
|
|
const _req = {
|
|
url: req.url,
|
|
method: req.method,
|
|
headers: iterableToObject(req.headers)
|
|
}
|
|
const _res = {
|
|
status: res.status,
|
|
headers: iterableToObject(res.headers)
|
|
}
|
|
|
|
return new CachePolicy(_req, _res, { shared: false })
|
|
}
|
|
|
|
// https://tools.ietf.org/html/rfc7234#section-4.2
|
|
function isStale (req, res) {
|
|
if (!res) {
|
|
return null
|
|
}
|
|
|
|
const _req = {
|
|
url: req.url,
|
|
method: req.method,
|
|
headers: iterableToObject(req.headers)
|
|
}
|
|
|
|
const policy = makePolicy(req, res)
|
|
|
|
const responseTime = res.headers.get('x-local-cache-time') ||
|
|
res.headers.get('date') ||
|
|
0
|
|
|
|
policy._responseTime = new Date(responseTime)
|
|
|
|
const bool = !policy.satisfiesWithoutRevalidation(_req)
|
|
return bool
|
|
}
|
|
|
|
function mustRevalidate (res) {
|
|
return (res.headers.get('cache-control') || '').match(/must-revalidate/i)
|
|
}
|
|
|
|
function conditionalFetch (req, cachedRes, opts) {
|
|
const _req = {
|
|
url: req.url,
|
|
method: req.method,
|
|
headers: Object.assign({}, opts.headers || {})
|
|
}
|
|
|
|
const policy = makePolicy(req, cachedRes)
|
|
opts.headers = policy.revalidationHeaders(_req)
|
|
|
|
return remoteFetch(req.url, opts)
|
|
.then(condRes => {
|
|
const revalidatedPolicy = policy.revalidatedPolicy(_req, {
|
|
status: condRes.status,
|
|
headers: iterableToObject(condRes.headers)
|
|
})
|
|
|
|
if (condRes.status >= 500 && !mustRevalidate(cachedRes)) {
|
|
// 111 Revalidation failed
|
|
// MUST be included if a cache returns a stale response because an
|
|
// attempt to revalidate the response failed, due to an inability to
|
|
// reach the server.
|
|
// (https://tools.ietf.org/html/rfc2616#section-14.46)
|
|
setWarning(cachedRes, 111, 'Revalidation failed')
|
|
return cachedRes
|
|
}
|
|
|
|
if (condRes.status === 304) { // 304 Not Modified
|
|
condRes.body = cachedRes.body
|
|
return opts.cacheManager.put(req, condRes, opts)
|
|
.then(newRes => {
|
|
newRes.headers = new fetch.Headers(revalidatedPolicy.policy.responseHeaders())
|
|
return newRes
|
|
})
|
|
}
|
|
|
|
return condRes
|
|
})
|
|
.then(res => res)
|
|
.catch(err => {
|
|
if (mustRevalidate(cachedRes)) {
|
|
throw err
|
|
} else {
|
|
// 111 Revalidation failed
|
|
// MUST be included if a cache returns a stale response because an
|
|
// attempt to revalidate the response failed, due to an inability to
|
|
// reach the server.
|
|
// (https://tools.ietf.org/html/rfc2616#section-14.46)
|
|
setWarning(cachedRes, 111, 'Revalidation failed')
|
|
// 199 Miscellaneous warning
|
|
// The warning text MAY include arbitrary information to be presented to
|
|
// a human user, or logged. A system receiving this warning MUST NOT take
|
|
// any automated action, besides presenting the warning to the user.
|
|
// (https://tools.ietf.org/html/rfc2616#section-14.46)
|
|
setWarning(
|
|
cachedRes,
|
|
199,
|
|
`Miscellaneous Warning ${err.code}: ${err.message}`
|
|
)
|
|
|
|
return cachedRes
|
|
}
|
|
})
|
|
}
|
|
|
|
function remoteFetchHandleIntegrity (res, integrity) {
|
|
const oldBod = res.body
|
|
const newBod = ssri.integrityStream({
|
|
integrity
|
|
})
|
|
oldBod.pipe(newBod)
|
|
res.body = newBod
|
|
oldBod.once('error', err => {
|
|
newBod.emit('error', err)
|
|
})
|
|
newBod.once('error', err => {
|
|
oldBod.emit('error', err)
|
|
})
|
|
}
|
|
|
|
function remoteFetch (uri, opts) {
|
|
const agent = getAgent(uri, opts)
|
|
const headers = Object.assign({
|
|
'connection': agent ? 'keep-alive' : 'close',
|
|
'user-agent': USER_AGENT
|
|
}, opts.headers || {})
|
|
|
|
const reqOpts = {
|
|
agent,
|
|
body: opts.body,
|
|
compress: opts.compress,
|
|
follow: opts.follow,
|
|
headers: new fetch.Headers(headers),
|
|
method: opts.method,
|
|
redirect: 'manual',
|
|
size: opts.size,
|
|
counter: opts.counter,
|
|
timeout: opts.timeout
|
|
}
|
|
|
|
return retry(
|
|
(retryHandler, attemptNum) => {
|
|
const req = new fetch.Request(uri, reqOpts)
|
|
return fetch(req)
|
|
.then(res => {
|
|
res.headers.set('x-fetch-attempts', attemptNum)
|
|
|
|
if (opts.integrity) {
|
|
remoteFetchHandleIntegrity(res, opts.integrity)
|
|
}
|
|
|
|
const isStream = req.body instanceof Stream
|
|
|
|
if (opts.cacheManager) {
|
|
const isMethodGetHead = req.method === 'GET' ||
|
|
req.method === 'HEAD'
|
|
|
|
const isCachable = opts.cache !== 'no-store' &&
|
|
isMethodGetHead &&
|
|
makePolicy(req, res).storable() &&
|
|
res.status === 200 // No other statuses should be stored!
|
|
|
|
if (isCachable) {
|
|
return opts.cacheManager.put(req, res, opts)
|
|
}
|
|
|
|
if (!isMethodGetHead) {
|
|
return opts.cacheManager.delete(req).then(() => {
|
|
if (res.status >= 500 && req.method !== 'POST' && !isStream) {
|
|
if (typeof opts.onRetry === 'function') {
|
|
opts.onRetry(res)
|
|
}
|
|
|
|
return retryHandler(res)
|
|
}
|
|
|
|
return res
|
|
})
|
|
}
|
|
}
|
|
|
|
const isRetriable = req.method !== 'POST' &&
|
|
!isStream && (
|
|
res.status === 408 || // Request Timeout
|
|
res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit)
|
|
res.status === 429 || // Too Many Requests ("standard" rate-limiting)
|
|
res.status >= 500 // Assume server errors are momentary hiccups
|
|
)
|
|
|
|
if (isRetriable) {
|
|
if (typeof opts.onRetry === 'function') {
|
|
opts.onRetry(res)
|
|
}
|
|
|
|
return retryHandler(res)
|
|
}
|
|
|
|
if (!fetch.isRedirect(res.status) || opts.redirect === 'manual') {
|
|
return res
|
|
}
|
|
|
|
// handle redirects - matches behavior of npm-fetch: https://github.com/bitinn/node-fetch
|
|
if (opts.redirect === 'error') {
|
|
const err = new Error(`redirect mode is set to error: ${uri}`)
|
|
err.code = 'ENOREDIRECT'
|
|
throw err
|
|
}
|
|
|
|
if (!res.headers.get('location')) {
|
|
const err = new Error(`redirect location header missing at: ${uri}`)
|
|
err.code = 'EINVALIDREDIRECT'
|
|
throw err
|
|
}
|
|
|
|
if (req.counter >= req.follow) {
|
|
const err = new Error(`maximum redirect reached at: ${uri}`)
|
|
err.code = 'EMAXREDIRECT'
|
|
throw err
|
|
}
|
|
|
|
const resolvedUrl = url.resolve(req.url, res.headers.get('location'))
|
|
let redirectURL = url.parse(resolvedUrl)
|
|
|
|
if (isURL.test(res.headers.get('location'))) {
|
|
redirectURL = url.parse(res.headers.get('location'))
|
|
}
|
|
|
|
// Remove authorization if changing hostnames (but not if just
|
|
// changing ports or protocols). This matches the behavior of request:
|
|
// https://github.com/request/request/blob/b12a6245/lib/redirect.js#L134-L138
|
|
if (url.parse(req.url).hostname !== redirectURL.hostname) {
|
|
req.headers.delete('authorization')
|
|
}
|
|
|
|
// for POST request with 301/302 response, or any request with 303 response,
|
|
// use GET when following redirect
|
|
if (res.status === 303 ||
|
|
((res.status === 301 || res.status === 302) && req.method === 'POST')) {
|
|
opts.method = 'GET'
|
|
opts.body = null
|
|
req.headers.delete('content-length')
|
|
}
|
|
|
|
opts.headers = {}
|
|
req.headers.forEach((value, name) => {
|
|
opts.headers[name] = value
|
|
})
|
|
|
|
opts.counter = ++req.counter
|
|
return cachingFetch(resolvedUrl, opts)
|
|
})
|
|
.catch(err => {
|
|
const code = err.code === 'EPROMISERETRY' ? err.retried.code : err.code
|
|
|
|
const isRetryError = RETRY_ERRORS.indexOf(code) === -1 &&
|
|
RETRY_TYPES.indexOf(err.type) === -1
|
|
|
|
if (req.method === 'POST' || isRetryError) {
|
|
throw err
|
|
}
|
|
|
|
if (typeof opts.onRetry === 'function') {
|
|
opts.onRetry(err)
|
|
}
|
|
|
|
return retryHandler(err)
|
|
})
|
|
},
|
|
opts.retry
|
|
).catch(err => {
|
|
if (err.status >= 400) {
|
|
return err
|
|
}
|
|
|
|
throw err
|
|
})
|
|
}
|
|
|
|
function isHeaderConditional (headers) {
|
|
if (!headers || typeof headers !== 'object') {
|
|
return false
|
|
}
|
|
|
|
const modifiers = [
|
|
'if-modified-since',
|
|
'if-none-match',
|
|
'if-unmodified-since',
|
|
'if-match',
|
|
'if-range'
|
|
]
|
|
|
|
return Object.keys(headers)
|
|
.some(h => modifiers.indexOf(h.toLowerCase()) !== -1)
|
|
}
|