mirror of
https://github.com/joelwmale/webhook-action.git
synced 2026-09-02 18:56:15 +08:00
Add node_modules until bundler is added
This commit is contained in:
Joel Male
+56
@@ -0,0 +1,56 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = {
|
||||
rules: {
|
||||
indent: [2, 4, {
|
||||
SwitchCase: 1
|
||||
}],
|
||||
quotes: [2, 'single'],
|
||||
'linebreak-style': [2, 'unix'],
|
||||
semi: [2, 'always'],
|
||||
strict: [2, 'global'],
|
||||
eqeqeq: 2,
|
||||
'dot-notation': 2,
|
||||
curly: 2,
|
||||
'no-fallthrough': 2,
|
||||
'quote-props': [2, 'as-needed'],
|
||||
'no-unused-expressions': [2, {
|
||||
allowShortCircuit: true
|
||||
}],
|
||||
'no-unused-vars': 2,
|
||||
'no-undef': 2,
|
||||
'handle-callback-err': 2,
|
||||
'no-new': 2,
|
||||
'new-cap': 2,
|
||||
'no-eval': 2,
|
||||
'no-invalid-this': 2,
|
||||
radix: [2, 'always'],
|
||||
'no-use-before-define': [2, 'nofunc'],
|
||||
'callback-return': [2, ['callback', 'cb', 'done']],
|
||||
'comma-dangle': [2, 'never'],
|
||||
'comma-style': [2, 'last'],
|
||||
'no-regex-spaces': 2,
|
||||
'no-empty': 2,
|
||||
'no-duplicate-case': 2,
|
||||
'no-empty-character-class': 2,
|
||||
'no-redeclare': [2, {
|
||||
builtinGlobals: true
|
||||
}],
|
||||
'block-scoped-var': 2,
|
||||
'no-sequences': 2,
|
||||
'no-throw-literal': 2,
|
||||
'no-useless-concat': 2,
|
||||
'no-void': 2,
|
||||
yoda: 2,
|
||||
'no-bitwise': 2,
|
||||
'no-lonely-if': 2,
|
||||
'no-mixed-spaces-and-tabs': 2,
|
||||
'no-console': 0
|
||||
},
|
||||
env: {
|
||||
es6: false,
|
||||
node: true
|
||||
},
|
||||
extends: 'eslint:recommended',
|
||||
fix: true
|
||||
};
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
.DS_Store
|
||||
node_modules
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
language: node_js
|
||||
sudo: false
|
||||
node_js:
|
||||
- "0.10"
|
||||
- 0.12
|
||||
- iojs
|
||||
- 4
|
||||
- 5
|
||||
before_install:
|
||||
- npm install -g grunt-cli
|
||||
notifications:
|
||||
email:
|
||||
- andris@kreata.ee
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
'use strict';
|
||||
|
||||
module.exports = function (grunt) {
|
||||
|
||||
// Project configuration.
|
||||
grunt.initConfig({
|
||||
eslint: {
|
||||
all: ['lib/*.js', 'test/*.js', 'Gruntfile.js', '.eslintrc.js']
|
||||
},
|
||||
|
||||
mochaTest: {
|
||||
all: {
|
||||
options: {
|
||||
reporter: 'spec'
|
||||
},
|
||||
src: ['test/*-test.js']
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Load the plugin(s)
|
||||
grunt.loadNpmTasks('grunt-eslint');
|
||||
grunt.loadNpmTasks('grunt-mocha-test');
|
||||
|
||||
// Tasks
|
||||
grunt.registerTask('default', ['eslint', 'mochaTest']);
|
||||
};
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
Copyright (c) 2011 Andris Reinman
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
# fetch
|
||||
|
||||
Fetch url contents. Supports gzipped content for quicker download, redirects (with automatic cookie handling, so no eternal redirect loops), streaming and piping etc.
|
||||
|
||||
[](https://travis-ci.org/andris9/fetch)
|
||||
|
||||
## Install
|
||||
|
||||
npm install fetch
|
||||
|
||||
## Usage
|
||||
|
||||
See examples folder for a complete example
|
||||
|
||||
## Fetch from URL
|
||||
|
||||
`fetch.fetchUrl(url [, options], callback)`
|
||||
|
||||
Where
|
||||
|
||||
* **url** is the url to fetch
|
||||
* **options** is an optional options object
|
||||
* **callback** is the callback to run - `callback(error, meta, body)`
|
||||
|
||||
Example
|
||||
|
||||
var fetchUrl = require("fetch").fetchUrl;
|
||||
|
||||
// source file is iso-8859-15 but it is converted to utf-8 automatically
|
||||
fetchUrl("http://kreata.ee/iso-8859-15.php", function(error, meta, body){
|
||||
console.log(body.toString());
|
||||
});
|
||||
|
||||
**NB** If the file has been marked with charset other than utf-8, it is converted automatically.
|
||||
|
||||
By default `iconv-lite` is used for charset conversion. If you want to use `node-iconv` module instead,
|
||||
add `"iconv": "*"` to your package.json file, it will be picked up by `fetch` automatically.
|
||||
|
||||
## Streaming
|
||||
|
||||
`fetch.FetchStream(url [, options]) -> Stream`
|
||||
|
||||
Where
|
||||
|
||||
* **url** is the url to fetch
|
||||
* **options** is an optional options object
|
||||
|
||||
With events:
|
||||
|
||||
* **data** with a data chunk - `function(chunk){}`
|
||||
* **meta** with some information about the response `function(meta){}`
|
||||
* **end** when the receiving is ready
|
||||
* **error**
|
||||
|
||||
Example
|
||||
|
||||
var FetchStream = require("fetch").FetchStream;
|
||||
|
||||
var fetch = new FetchStream("http://google.com");
|
||||
|
||||
fetch.on("data", function(chunk){
|
||||
console.log(chunk);
|
||||
});
|
||||
|
||||
## Options
|
||||
|
||||
Possible option values
|
||||
|
||||
* **maxRedirects** how many redirects allowed, defaults to 10
|
||||
* **disableRedirects** set to true if redirects are not allowed, defaults to false
|
||||
* **headers** optional header fields, in the form of `{'Header-Field':'value'}`
|
||||
* **maxResponseLength** maximum allowd length for the file, the remainder is cut off. Defaults to `Infinity`
|
||||
* **method** defaults to GET
|
||||
* **payload** request body
|
||||
* **disableGzip** set to false, to disable content gzipping, needed for Node v0.5.9 which has buggy zlib
|
||||
* **cookies** an array of cookie definitions in the form of `['name=val']`
|
||||
* **cookieJar** for sharing cookies between requests, see below
|
||||
* **outputEncoding** valid for `fetchUrl`
|
||||
* **disableDecoding** valid for `fetchUrl`, set to true to disable automatic charset decoding to utf-8
|
||||
* **overrideCharset** valid for `fetchUrl`, set input encoding
|
||||
* **asyncDnsLoookup** use high performance asyncronous DNS resolution based on c-ares instead of a thread pool calling getaddrinfo(3)
|
||||
* **timeout** set a timeout in ms
|
||||
* **agentHttps** pass-through http.request agent parameter for https
|
||||
* **agentHttp** pass-through http.request agent parameter for http
|
||||
* **agent** pass-through http.request agent parameter as fallback, if agentHttps or agentHttp are not specified
|
||||
* **rejectUnauthorized** whether to reject self-signed certificates (`true`, default behavior), or ignore and allow them (`false`)
|
||||
|
||||
|
||||
## Meta object
|
||||
|
||||
Meta object contains following fields:
|
||||
|
||||
* **status** HTTP status code
|
||||
* **responseHeaders** response headers
|
||||
* **finalUrl** last url value, useful with redirects
|
||||
* **redirectCount** how many redirects happened
|
||||
* **cookieJar** CookieJar object for sharing/retrieving cookies
|
||||
|
||||
## Headers
|
||||
|
||||
Request headers can be set with `options.headers`
|
||||
|
||||
options = {
|
||||
headers:{
|
||||
"X-My-Header": "This is a custom header field"
|
||||
}
|
||||
}
|
||||
|
||||
## User-Agent
|
||||
User-Agent value can be set with `options.headers['User-Agent']` value. Defaults to `"FetchStream"`
|
||||
|
||||
options = {
|
||||
headers: {
|
||||
"User-Agent": "MyUseragent/1.0"
|
||||
}
|
||||
}
|
||||
|
||||
## Cookies
|
||||
Cookies can be set with `options.cookies` which takes an array with cookie definitions
|
||||
|
||||
options = {
|
||||
cookie: ["name=value", "key=value; path=/; secure"]
|
||||
}
|
||||
|
||||
Paths, domain, expire and other cookie settings are honored, so try not to set cookies with expire dates in the past. If domain is not set, any domain will pass, same for paths.
|
||||
|
||||
**NB** Do not set cookie field directly in request header as it will be overwritten.
|
||||
|
||||
## Cookie sharing
|
||||
|
||||
Cookies can be shared between different requests, this can be achieved with `CookieJar`
|
||||
|
||||
var fetch = require("fetch");
|
||||
|
||||
var cookies = new fetch.CookieJar();
|
||||
|
||||
// add one cookie for testing
|
||||
cookies.setCookie('alfa=beta; path=/;');
|
||||
|
||||
// create a FetchStream with custom CookieJar
|
||||
var f = fetch.FetchStream("http://www.example.com/page1",{cookieJar: cookies});
|
||||
|
||||
f.on("end", function(){
|
||||
// if cookies were set with the previos request, the data is
|
||||
// saved in 'cookieJar' and passed to the next request
|
||||
fetch.FetchStream("http://www.example.com/page1",{cookieJar: cookies});
|
||||
});
|
||||
|
||||
|
||||
## Redirects
|
||||
|
||||
Redirects are on by default, use `options.disableRedirects` to disable. Maximum redirect count can be set with `options.maxRedirects` (defaults to 10)
|
||||
|
||||
options = {
|
||||
disableRedirects: true
|
||||
}
|
||||
|
||||
options = {
|
||||
maxRedirects: 100
|
||||
}
|
||||
|
||||
## Disable Gzip support
|
||||
|
||||
Gzip and Deflate support is automatically on. This is problematic in Node v0.5.9 and below since Zlib support on these versions is buggy with unpacking and tends to yield in error.
|
||||
|
||||
options = {
|
||||
disableGzip: true
|
||||
}
|
||||
|
||||
## Piping to file
|
||||
|
||||
`FetchStream` is a readable Stream object and thus can be piped. For example stream URL contents directly to a file:
|
||||
|
||||
var FetchStream = require("fetch").FetchStream,
|
||||
fs = require("fs"),
|
||||
out;
|
||||
|
||||
out = fs.createWriteStream('file.html');
|
||||
new FetchStream("http://www.example.com/index.php").pipe(out);
|
||||
|
||||
## License
|
||||
|
||||
BSD
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
Caching in HTTP
|
||||
===============
|
||||
|
||||
ETAG
|
||||
----
|
||||
|
||||
Res 200:
|
||||
|
||||
ETag: "fa6e-3e3073913b100"
|
||||
|
||||
Req:
|
||||
|
||||
If-None-Match: "fa6e-3e3073913b100"
|
||||
|
||||
Res 304:
|
||||
|
||||
ETag: "fa6e-3e3073913b100"
|
||||
|
||||
|
||||
LAST MODIFIED
|
||||
-------------
|
||||
|
||||
Res 200:
|
||||
|
||||
Last-Modified: Mon, 28 Jan 2013 22:29:45 GMT
|
||||
|
||||
Req:
|
||||
|
||||
If-Modified-Since: Mon, 28 Jan 2013 22:29:45 GMT
|
||||
|
||||
Res 304:
|
||||
|
||||
Last-Modified:Mon, 28 Jan 2013 22:29:45 GMT
|
||||
|
||||
|
||||
EXPIRES
|
||||
-------
|
||||
|
||||
Res 200:
|
||||
|
||||
Expires: Tue, 19 Mar 2013 11:17:57 GMT
|
||||
|
||||
Do not try again before Tue, 19 Mar 2013 11:17:57 GMT
|
||||
|
||||
CACHE-CONTROL
|
||||
-------------
|
||||
|
||||
Res 200:
|
||||
|
||||
Cache-Control: max-age=300
|
||||
|
||||
Do not try again until 5 minutes from now
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
'use strict';
|
||||
|
||||
var FetchStream = require('../lib/fetch').FetchStream;
|
||||
|
||||
var fetch = new FetchStream('http://google.com', {
|
||||
headers: {}
|
||||
});
|
||||
|
||||
fetch.on('data', function (chunk) {
|
||||
console.log(chunk);
|
||||
});
|
||||
|
||||
fetch.on('meta', function (meta) {
|
||||
console.log(meta);
|
||||
});
|
||||
|
||||
fetch.on('end', function () {
|
||||
console.log('END');
|
||||
});
|
||||
|
||||
fetch.on('error', function (e) {
|
||||
console.log('ERROR: ' + (e && e.message || e));
|
||||
});
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
'use strict';
|
||||
|
||||
// fetch url and update charset to utf-8
|
||||
var fetchUrl = require('../lib/fetch').fetchUrl;
|
||||
|
||||
|
||||
fetchUrl('http://kreata.ee/iso-8859-15.php', function(error, meta, body){
|
||||
if(error){
|
||||
return console.log('ERROR', error.message || error);
|
||||
}
|
||||
|
||||
console.log('META INFO');
|
||||
console.log(meta);
|
||||
|
||||
console.log('BODY');
|
||||
console.log(body.toString('utf-8'));
|
||||
});
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
'use strict';
|
||||
|
||||
// pipe to file
|
||||
|
||||
var FetchStream = require('../lib/fetch').FetchStream,
|
||||
fs = require('fs'),
|
||||
inp, out;
|
||||
|
||||
inp = new FetchStream('http://google.com');
|
||||
out = fs.createWriteStream('google.html');
|
||||
|
||||
inp.on('end', function(){
|
||||
console.log('downloaded!');
|
||||
});
|
||||
|
||||
inp.pipe(out);
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
'use strict';
|
||||
|
||||
var urllib = require('url');
|
||||
|
||||
module.exports.CookieJar = CookieJar;
|
||||
|
||||
function CookieJar(options) {
|
||||
this.options = options || {};
|
||||
this.options.sessionTimeout = this.options.sessionTimeout || 1800; // 30min
|
||||
|
||||
this.cookies = {};
|
||||
}
|
||||
|
||||
CookieJar.prototype.addCookie = function (cookie) {
|
||||
|
||||
if (!cookie || !cookie.name) {
|
||||
return;
|
||||
}
|
||||
|
||||
var lcookie;
|
||||
|
||||
if (!this.cookies[cookie.name]) {
|
||||
this.cookies[cookie.name] = [];
|
||||
}
|
||||
|
||||
// overwrite if has same params
|
||||
for (var i = 0, len = this.cookies[cookie.name].length; i < len; i++) {
|
||||
lcookie = this.cookies[cookie.name][i];
|
||||
if (
|
||||
lcookie.path === cookie.path &&
|
||||
lcookie.domain === cookie.domain &&
|
||||
lcookie.secure === cookie.secure &&
|
||||
lcookie.httponly === cookie.httponly
|
||||
) {
|
||||
this.cookies[cookie.name][i] = cookie;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
this.cookies[cookie.name].push(cookie);
|
||||
};
|
||||
|
||||
CookieJar.prototype.getCookies = function (url) {
|
||||
var keys = Object.keys(this.cookies),
|
||||
cookie, cookies = [];
|
||||
|
||||
for (var i = 0, len = keys.length; i < len; i++) {
|
||||
if (Array.isArray(this.cookies[keys[i]])) {
|
||||
for (var j = 0, lenj = this.cookies[keys[i]].length; j < lenj; j++) {
|
||||
cookie = this.cookies[keys[i]][j];
|
||||
if (this.matchCookie(cookie, url)) {
|
||||
cookies.push(cookie.name + '=' + cookie.value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return cookies.join('; ');
|
||||
};
|
||||
|
||||
CookieJar.prototype.matchCookie = function (cookie, url) {
|
||||
var urlparts = urllib.parse(url || '', false, true),
|
||||
path;
|
||||
|
||||
// check expire
|
||||
if (cookie.expire) {
|
||||
if (cookie.expire.getTime() < Date.now()) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// check if hostname matches
|
||||
if (urlparts.hostname && cookie._domain) {
|
||||
if (!(urlparts.hostname === cookie._domain || urlparts.hostname.substr(-(cookie._domain.length + 1)) === '.' + cookie._domain)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// check if path matches
|
||||
if (cookie.path && urlparts.pathname) {
|
||||
|
||||
path = (urlparts.pathname || '/').split('/');
|
||||
path.pop();
|
||||
path = path.join('/').trim();
|
||||
if (path.substr(0, 1) !== '/') {
|
||||
path = '/' + path;
|
||||
}
|
||||
if (path.substr(-1) !== '/') {
|
||||
path += '/';
|
||||
}
|
||||
|
||||
if (path.substr(0, cookie.path.length) !== cookie.path) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// check secure
|
||||
if (cookie.secure && urlparts.protocol) {
|
||||
if (urlparts.protocol !== 'https:') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// check httponly
|
||||
if (cookie.httponly && urlparts.protocol) {
|
||||
if (urlparts.protocol !== 'http:') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
CookieJar.prototype.setCookie = function (cookie_str, url) {
|
||||
var parts = (cookie_str || '').split(';'),
|
||||
cookie = {},
|
||||
urlparts = urllib.parse(url || '', false, true),
|
||||
path;
|
||||
|
||||
parts.forEach((function (part) {
|
||||
var key, val;
|
||||
part = part.split('=');
|
||||
key = part.shift().trim();
|
||||
val = part.join('=').trim();
|
||||
|
||||
if (!key) {
|
||||
return;
|
||||
}
|
||||
|
||||
switch (key.toLowerCase()) {
|
||||
|
||||
case 'expires':
|
||||
|
||||
cookie.expires = new Date(val);
|
||||
break;
|
||||
|
||||
case 'path':
|
||||
cookie.path = val.trim();
|
||||
break;
|
||||
|
||||
case 'domain':
|
||||
cookie.domain = val.toLowerCase();
|
||||
break;
|
||||
|
||||
case 'max-age':
|
||||
cookie.expires = new Date(Date.now() + (Number(val) || 0) * 1000);
|
||||
break;
|
||||
|
||||
case 'secure':
|
||||
cookie.secure = true;
|
||||
break;
|
||||
|
||||
case 'httponly':
|
||||
cookie.httponly = true;
|
||||
break;
|
||||
|
||||
default:
|
||||
if (!cookie.name) {
|
||||
cookie.name = key;
|
||||
cookie.value = val;
|
||||
}
|
||||
}
|
||||
}).bind(this));
|
||||
|
||||
// use current path when path is not specified
|
||||
if (!cookie.path) {
|
||||
path = (urlparts.pathname || '/').split('/');
|
||||
path.pop();
|
||||
cookie.path = path.join('/').trim();
|
||||
if (cookie.path.substr(0, 1) !== '/') {
|
||||
cookie.path = '/' + cookie.path;
|
||||
}
|
||||
if (cookie.path.substr(-1) !== '/') {
|
||||
cookie.path += '/';
|
||||
}
|
||||
}
|
||||
|
||||
// if no expire date, then use sessionTimeout value
|
||||
if (!cookie.expires) {
|
||||
cookie._expires = new Date(Date.now() + (Number(this.options.sessionTimeout) || 0) * 1000);
|
||||
} else {
|
||||
cookie._expires = cookie.expires;
|
||||
}
|
||||
|
||||
if (!cookie.domain) {
|
||||
if (urlparts.hostname) {
|
||||
cookie._domain = urlparts.hostname;
|
||||
}
|
||||
} else {
|
||||
cookie._domain = cookie.domain;
|
||||
}
|
||||
|
||||
this.addCookie(cookie);
|
||||
};
|
||||
+500
@@ -0,0 +1,500 @@
|
||||
'use strict';
|
||||
|
||||
var http = require('http');
|
||||
var https = require('https');
|
||||
var urllib = require('url');
|
||||
var utillib = require('util');
|
||||
var zlib = require('zlib');
|
||||
var dns = require('dns');
|
||||
var Stream = require('stream').Readable;
|
||||
var CookieJar = require('./cookiejar').CookieJar;
|
||||
var encodinglib = require('encoding');
|
||||
var net = require('net');
|
||||
|
||||
var USE_ALLOC = typeof Buffer.alloc === 'function';
|
||||
|
||||
exports.FetchStream = FetchStream;
|
||||
exports.CookieJar = CookieJar;
|
||||
exports.fetchUrl = fetchUrl;
|
||||
|
||||
function FetchStream(url, options) {
|
||||
Stream.call(this);
|
||||
|
||||
options = options || {};
|
||||
|
||||
this.url = url;
|
||||
if (!this.url) {
|
||||
return this.emit('error', new Error('url not defined'));
|
||||
}
|
||||
|
||||
this.userAgent = options.userAgent || 'FetchStream';
|
||||
|
||||
this._redirect_count = 0;
|
||||
|
||||
this.options = options || {};
|
||||
this.normalizeOptions();
|
||||
|
||||
// prevent errors before 'error' handler is set by defferring actions
|
||||
if (typeof setImmediate !== 'undefined') {
|
||||
setImmediate(this.runStream.bind(this, url));
|
||||
} else {
|
||||
process.nextTick(this.runStream.bind(this, url));
|
||||
}
|
||||
this.responseBuffer = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');
|
||||
this.ended = false;
|
||||
this.readyToRead = 0;
|
||||
}
|
||||
utillib.inherits(FetchStream, Stream);
|
||||
|
||||
FetchStream.prototype._read = function (size) {
|
||||
if (this.ended && this.responseBuffer.length === 0) {
|
||||
this.push(null);
|
||||
return;
|
||||
}
|
||||
this.readyToRead += size;
|
||||
this.drainBuffer();
|
||||
};
|
||||
|
||||
FetchStream.prototype.drainBuffer = function () {
|
||||
if (this.readyToRead === 0) {
|
||||
return;
|
||||
}
|
||||
if (this.responseBuffer.length === 0) {
|
||||
return;
|
||||
}
|
||||
var push;
|
||||
var rest;
|
||||
var restSize;
|
||||
|
||||
if (this.responseBuffer.length > this.readyToRead) {
|
||||
push = USE_ALLOC ? Buffer.alloc(this.readyToRead, '', 'binary') : new Buffer(this.readyToRead, 'binary');
|
||||
this.responseBuffer.copy(push, 0, 0, this.readyToRead);
|
||||
restSize = this.responseBuffer.length - this.readyToRead;
|
||||
rest = USE_ALLOC ? Buffer.alloc(restSize, '', 'binary') : new Buffer(restSize, 'binary');
|
||||
this.responseBuffer.copy(rest, 0, this.readyToRead);
|
||||
} else {
|
||||
push = this.responseBuffer;
|
||||
rest = USE_ALLOC ? Buffer.alloc(0, '', 'binary') : new Buffer(0, 'binary');
|
||||
}
|
||||
this.responseBuffer = rest;
|
||||
this.readyToRead = 0;
|
||||
if (this.options.encoding) {
|
||||
this.push(push, this.options.encoding);
|
||||
} else {
|
||||
this.push(push);
|
||||
}
|
||||
};
|
||||
|
||||
FetchStream.prototype.destroy = function (ex) {
|
||||
this.emit('destroy', ex);
|
||||
};
|
||||
|
||||
FetchStream.prototype.normalizeOptions = function () {
|
||||
|
||||
// cookiejar
|
||||
this.cookieJar = this.options.cookieJar || new CookieJar();
|
||||
|
||||
// default redirects - 10
|
||||
// if disableRedirect is set, then 0
|
||||
if (!this.options.disableRedirect && typeof this.options.maxRedirects !== 'number' &&
|
||||
!(this.options.maxRedirects instanceof Number)) {
|
||||
this.options.maxRedirects = 10;
|
||||
} else if (this.options.disableRedirects) {
|
||||
this.options.maxRedirects = 0;
|
||||
}
|
||||
|
||||
// normalize header keys
|
||||
// HTTP and HTTPS takes in key names in case insensitive but to find
|
||||
// an exact value from an object key name needs to be case sensitive
|
||||
// so we're just lowercasing all input keys
|
||||
this.options.headers = this.options.headers || {};
|
||||
|
||||
var keys = Object.keys(this.options.headers);
|
||||
var newheaders = {};
|
||||
var i;
|
||||
|
||||
for (i = keys.length - 1; i >= 0; i--) {
|
||||
newheaders[keys[i].toLowerCase().trim()] = this.options.headers[keys[i]];
|
||||
}
|
||||
|
||||
this.options.headers = newheaders;
|
||||
|
||||
if (!this.options.headers['user-agent']) {
|
||||
this.options.headers['user-agent'] = this.userAgent;
|
||||
}
|
||||
|
||||
if (!this.options.headers.pragma) {
|
||||
this.options.headers.pragma = 'no-cache';
|
||||
}
|
||||
|
||||
if (!this.options.headers['cache-control']) {
|
||||
this.options.headers['cache-control'] = 'no-cache';
|
||||
}
|
||||
|
||||
if (!this.options.disableGzip) {
|
||||
this.options.headers['accept-encoding'] = 'gzip, deflate';
|
||||
} else {
|
||||
delete this.options.headers['accept-encoding'];
|
||||
}
|
||||
|
||||
// max length for the response,
|
||||
// if not set, default is Infinity
|
||||
if (!this.options.maxResponseLength) {
|
||||
this.options.maxResponseLength = Infinity;
|
||||
}
|
||||
|
||||
// method:
|
||||
// defaults to GET, or when payload present to POST
|
||||
if (!this.options.method) {
|
||||
this.options.method = this.options.payload || this.options.payloadSize ? 'POST' : 'GET';
|
||||
}
|
||||
|
||||
// set cookies
|
||||
// takes full cookie definition strings as params
|
||||
if (this.options.cookies) {
|
||||
for (i = 0; i < this.options.cookies.length; i++) {
|
||||
this.cookieJar.setCookie(this.options.cookies[i], this.url);
|
||||
}
|
||||
}
|
||||
|
||||
// rejectUnauthorized
|
||||
if (typeof this.options.rejectUnauthorized === 'undefined') {
|
||||
this.options.rejectUnauthorized = true;
|
||||
}
|
||||
};
|
||||
|
||||
FetchStream.prototype.parseUrl = function (url) {
|
||||
var urlparts = urllib.parse(url, false, true),
|
||||
transport,
|
||||
urloptions = {
|
||||
host: urlparts.hostname || urlparts.host,
|
||||
port: urlparts.port,
|
||||
path: urlparts.pathname + (urlparts.search || '') || '/',
|
||||
method: this.options.method,
|
||||
rejectUnauthorized: this.options.rejectUnauthorized
|
||||
};
|
||||
|
||||
switch (urlparts.protocol) {
|
||||
case 'https:':
|
||||
transport = https;
|
||||
break;
|
||||
case 'http:':
|
||||
default:
|
||||
transport = http;
|
||||
break;
|
||||
}
|
||||
|
||||
if (transport === https) {
|
||||
if('agentHttps' in this.options){
|
||||
urloptions.agent = this.options.agentHttps;
|
||||
}
|
||||
if('agent' in this.options){
|
||||
urloptions.agent = this.options.agent;
|
||||
}
|
||||
} else {
|
||||
if('agentHttp' in this.options){
|
||||
urloptions.agent = this.options.agentHttp;
|
||||
}
|
||||
if('agent' in this.options){
|
||||
urloptions.agent = this.options.agent;
|
||||
}
|
||||
}
|
||||
|
||||
if (!urloptions.port) {
|
||||
switch (urlparts.protocol) {
|
||||
case 'https:':
|
||||
urloptions.port = 443;
|
||||
break;
|
||||
case 'http:':
|
||||
default:
|
||||
urloptions.port = 80;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
urloptions.headers = this.options.headers || {};
|
||||
|
||||
if (urlparts.auth) {
|
||||
var buf = USE_ALLOC ? Buffer.alloc(Buffer.byteLength(urlparts.auth), urlparts.auth) : new Buffer(urlparts.auth);
|
||||
urloptions.headers.Authorization = 'Basic ' + buf.toString('base64');
|
||||
}
|
||||
|
||||
return {
|
||||
urloptions: urloptions,
|
||||
transport: transport
|
||||
};
|
||||
};
|
||||
|
||||
FetchStream.prototype.setEncoding = function (encoding) {
|
||||
this.options.encoding = encoding;
|
||||
};
|
||||
|
||||
FetchStream.prototype.runStream = function (url) {
|
||||
var url_data = this.parseUrl(url),
|
||||
cookies = this.cookieJar.getCookies(url);
|
||||
|
||||
if (cookies) {
|
||||
url_data.urloptions.headers.cookie = cookies;
|
||||
} else {
|
||||
delete url_data.urloptions.headers.cookie;
|
||||
}
|
||||
|
||||
if (this.options.payload) {
|
||||
url_data.urloptions.headers['content-length'] = Buffer.byteLength(this.options.payload || '', 'utf-8');
|
||||
}
|
||||
|
||||
if (this.options.payloadSize) {
|
||||
url_data.urloptions.headers['content-length'] = this.options.payloadSize;
|
||||
}
|
||||
|
||||
if (this.options.asyncDnsLoookup) {
|
||||
var dnsCallback = (function (err, addresses) {
|
||||
if (err) {
|
||||
this.emit('error', err);
|
||||
return;
|
||||
}
|
||||
|
||||
url_data.urloptions.headers.host = url_data.urloptions.hostname || url_data.urloptions.host;
|
||||
url_data.urloptions.hostname = addresses[0];
|
||||
url_data.urloptions.host = url_data.urloptions.headers.host + (url_data.urloptions.port ? ':' + url_data.urloptions.port : '');
|
||||
|
||||
this._runStream(url_data, url);
|
||||
}).bind(this);
|
||||
|
||||
if (net.isIP(url_data.urloptions.host)) {
|
||||
dnsCallback(null, [url_data.urloptions.host]);
|
||||
} else {
|
||||
dns.resolve4(url_data.urloptions.host, dnsCallback);
|
||||
}
|
||||
} else {
|
||||
this._runStream(url_data, url);
|
||||
}
|
||||
};
|
||||
|
||||
FetchStream.prototype._runStream = function (url_data, url) {
|
||||
|
||||
var req = url_data.transport.request(url_data.urloptions, (function (res) {
|
||||
|
||||
// catch new cookies before potential redirect
|
||||
if (Array.isArray(res.headers['set-cookie'])) {
|
||||
for (var i = 0; i < res.headers['set-cookie'].length; i++) {
|
||||
this.cookieJar.setCookie(res.headers['set-cookie'][i], url);
|
||||
}
|
||||
}
|
||||
|
||||
if ([301, 302, 303, 307, 308].indexOf(res.statusCode) >= 0) {
|
||||
if (!this.options.disableRedirects && this.options.maxRedirects > this._redirect_count && res.headers.location) {
|
||||
this._redirect_count++;
|
||||
req.destroy();
|
||||
this.runStream(urllib.resolve(url, res.headers.location));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
this.meta = {
|
||||
status: res.statusCode,
|
||||
responseHeaders: res.headers,
|
||||
finalUrl: url,
|
||||
redirectCount: this._redirect_count,
|
||||
cookieJar: this.cookieJar
|
||||
};
|
||||
|
||||
var curlen = 0,
|
||||
maxlen,
|
||||
|
||||
receive = (function (chunk) {
|
||||
if (curlen + chunk.length > this.options.maxResponseLength) {
|
||||
maxlen = this.options.maxResponseLength - curlen;
|
||||
} else {
|
||||
maxlen = chunk.length;
|
||||
}
|
||||
|
||||
if (maxlen <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
curlen += Math.min(maxlen, chunk.length);
|
||||
if (maxlen >= chunk.length) {
|
||||
if (this.responseBuffer.length === 0) {
|
||||
this.responseBuffer = chunk;
|
||||
} else {
|
||||
this.responseBuffer = Buffer.concat([this.responseBuffer, chunk]);
|
||||
}
|
||||
} else {
|
||||
this.responseBuffer = Buffer.concat([this.responseBuffer, chunk], this.responseBuffer.length + maxlen);
|
||||
}
|
||||
this.drainBuffer();
|
||||
}).bind(this),
|
||||
|
||||
error = (function (e) {
|
||||
this.ended = true;
|
||||
this.emit('error', e);
|
||||
this.drainBuffer();
|
||||
}).bind(this),
|
||||
|
||||
end = (function () {
|
||||
this.ended = true;
|
||||
if (this.responseBuffer.length === 0) {
|
||||
this.push(null);
|
||||
}
|
||||
}).bind(this),
|
||||
|
||||
unpack = (function (type, res) {
|
||||
var z = zlib['create' + type]();
|
||||
z.on('data', receive);
|
||||
z.on('error', error);
|
||||
z.on('end', end);
|
||||
res.pipe(z);
|
||||
}).bind(this);
|
||||
|
||||
this.emit('meta', this.meta);
|
||||
|
||||
if (res.headers['content-encoding']) {
|
||||
switch (res.headers['content-encoding'].toLowerCase().trim()) {
|
||||
case 'gzip':
|
||||
return unpack('Gunzip', res);
|
||||
case 'deflate':
|
||||
return unpack('InflateRaw', res);
|
||||
}
|
||||
}
|
||||
|
||||
res.on('data', receive);
|
||||
res.on('end', end);
|
||||
|
||||
}).bind(this));
|
||||
|
||||
req.on('error', (function (e) {
|
||||
this.emit('error', e);
|
||||
}).bind(this));
|
||||
|
||||
if (this.options.timeout) {
|
||||
req.setTimeout(this.options.timeout, req.abort.bind(req));
|
||||
}
|
||||
this.on('destroy', req.abort.bind(req));
|
||||
|
||||
if (this.options.payload) {
|
||||
req.end(this.options.payload);
|
||||
} else if (this.options.payloadStream) {
|
||||
this.options.payloadStream.pipe(req);
|
||||
this.options.payloadStream.resume();
|
||||
} else {
|
||||
req.end();
|
||||
}
|
||||
};
|
||||
|
||||
function fetchUrl(url, options, callback) {
|
||||
if (!callback && typeof options === 'function') {
|
||||
callback = options;
|
||||
options = undefined;
|
||||
}
|
||||
options = options || {};
|
||||
|
||||
var fetchstream = new FetchStream(url, options),
|
||||
response_data, chunks = [],
|
||||
length = 0,
|
||||
curpos = 0,
|
||||
buffer,
|
||||
content_type,
|
||||
callbackFired = false;
|
||||
|
||||
fetchstream.on('meta', function (meta) {
|
||||
response_data = meta;
|
||||
content_type = _parseContentType(meta.responseHeaders['content-type']);
|
||||
});
|
||||
|
||||
fetchstream.on('data', function (chunk) {
|
||||
if (chunk) {
|
||||
chunks.push(chunk);
|
||||
length += chunk.length;
|
||||
}
|
||||
});
|
||||
|
||||
fetchstream.on('error', function (error) {
|
||||
if (error && error.code === 'HPE_INVALID_CONSTANT') {
|
||||
// skip invalid formatting errors
|
||||
return;
|
||||
}
|
||||
if (callbackFired) {
|
||||
return;
|
||||
}
|
||||
callbackFired = true;
|
||||
callback(error);
|
||||
});
|
||||
|
||||
fetchstream.on('end', function () {
|
||||
if (callbackFired) {
|
||||
return;
|
||||
}
|
||||
callbackFired = true;
|
||||
|
||||
buffer = USE_ALLOC ? Buffer.alloc(length) : new Buffer(length);
|
||||
for (var i = 0, len = chunks.length; i < len; i++) {
|
||||
chunks[i].copy(buffer, curpos);
|
||||
curpos += chunks[i].length;
|
||||
}
|
||||
|
||||
if (content_type.mimeType === 'text/html') {
|
||||
content_type.charset = _findHTMLCharset(buffer) || content_type.charset;
|
||||
}
|
||||
|
||||
content_type.charset = (options.overrideCharset || content_type.charset || 'utf-8').trim().toLowerCase();
|
||||
|
||||
|
||||
if (!options.disableDecoding && !content_type.charset.match(/^utf-?8$/i)) {
|
||||
buffer = encodinglib.convert(buffer, 'UTF-8', content_type.charset);
|
||||
}
|
||||
|
||||
if (options.outputEncoding) {
|
||||
return callback(null, response_data, buffer.toString(options.outputEncoding));
|
||||
} else {
|
||||
return callback(null, response_data, buffer);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
function _parseContentType(str) {
|
||||
if (!str) {
|
||||
return {};
|
||||
}
|
||||
var parts = str.split(';'),
|
||||
mimeType = parts.shift(),
|
||||
charset, chparts;
|
||||
|
||||
for (var i = 0, len = parts.length; i < len; i++) {
|
||||
chparts = parts[i].split('=');
|
||||
if (chparts.length > 1) {
|
||||
if (chparts[0].trim().toLowerCase() === 'charset') {
|
||||
charset = chparts[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
mimeType: (mimeType || '').trim().toLowerCase(),
|
||||
charset: (charset || 'UTF-8').trim().toLowerCase() // defaults to UTF-8
|
||||
};
|
||||
}
|
||||
|
||||
function _findHTMLCharset(htmlbuffer) {
|
||||
|
||||
var body = htmlbuffer.toString('ascii'),
|
||||
input, meta, charset;
|
||||
|
||||
if ((meta = body.match(/<meta\s+http-equiv=["']content-type["'][^>]*?>/i))) {
|
||||
input = meta[0];
|
||||
}
|
||||
|
||||
if (input) {
|
||||
charset = input.match(/charset\s?=\s?([a-zA-Z\-0-9]*);?/);
|
||||
if (charset) {
|
||||
charset = (charset[1] || '').trim().toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
if (!charset && (meta = body.match(/<meta\s+charset=["'](.*?)["']/i))) {
|
||||
charset = (meta[1] || '').trim().toLowerCase();
|
||||
}
|
||||
|
||||
return charset;
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name": "fetch",
|
||||
"description": "Fetch URL contents",
|
||||
"version": "1.1.0",
|
||||
"author": "Andris Reinman",
|
||||
"homepage": "http://github.com/andris9/fetch",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git://github.com/andris9/fetch.git"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "grunt"
|
||||
},
|
||||
"main": "./lib/fetch",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"biskviit": "1.0.1",
|
||||
"encoding": "0.1.12"
|
||||
},
|
||||
"devDependencies": {
|
||||
"chai": "^3.5.0",
|
||||
"grunt": "^1.0.1",
|
||||
"grunt-eslint": "^18.1.0",
|
||||
"grunt-mocha-test": "^0.12.7",
|
||||
"mocha": "^2.5.3"
|
||||
},
|
||||
"keywords": [
|
||||
"url"
|
||||
]
|
||||
}
|
||||
+293
@@ -0,0 +1,293 @@
|
||||
/* eslint no-unused-expressions:0 */
|
||||
/* globals afterEach, beforeEach, describe, it */
|
||||
|
||||
'use strict';
|
||||
|
||||
var chai = require('chai');
|
||||
var expect = chai.expect;
|
||||
|
||||
//var http = require('http');
|
||||
var fetch = require('../lib/fetch');
|
||||
var http = require('http');
|
||||
var https = require('https');
|
||||
|
||||
chai.config.includeStack = true;
|
||||
|
||||
var HTTP_PORT = 9998;
|
||||
var HTTPS_PORT = 9993;
|
||||
var USE_ALLOC = typeof Buffer.alloc === 'function';
|
||||
|
||||
var httpsOptions = {
|
||||
key: '-----BEGIN RSA PRIVATE KEY-----\n' +
|
||||
'MIIEpAIBAAKCAQEA6Z5Qqhw+oWfhtEiMHE32Ht94mwTBpAfjt3vPpX8M7DMCTwHs\n' +
|
||||
'1xcXvQ4lQ3rwreDTOWdoJeEEy7gMxXqH0jw0WfBx+8IIJU69xstOyT7FRFDvA1yT\n' +
|
||||
'RXY2yt9K5s6SKken/ebMfmZR+03ND4UFsDzkz0FfgcjrkXmrMF5Eh5UXX/+9YHeU\n' +
|
||||
'xlp0gMAt+/SumSmgCaysxZLjLpd4uXz+X+JVxsk1ACg1NoEO7lWJC/3WBP7MIcu2\n' +
|
||||
'wVsMd2XegLT0gWYfT1/jsIH64U/mS/SVXC9QhxMl9Yfko2kx1OiYhDxhHs75RJZh\n' +
|
||||
'rNRxgfiwgSb50Gw4NAQaDIxr/DJPdLhgnpY6UQIDAQABAoIBAE+tfzWFjJbgJ0ql\n' +
|
||||
's6Ozs020Sh4U8TZQuonJ4HhBbNbiTtdDgNObPK1uNadeNtgW5fOeIRdKN6iDjVeN\n' +
|
||||
'AuXhQrmqGDYVZ1HSGUfD74sTrZQvRlWPLWtzdhybK6Css41YAyPFo9k4bJ2ZW2b/\n' +
|
||||
'p4EEQ8WsNja9oBpttMU6YYUchGxo1gujN8hmfDdXUQx3k5Xwx4KA68dveJ8GasIt\n' +
|
||||
'd+0Jd/FVwCyyx8HTiF1FF8QZYQeAXxbXJgLBuCsMQJghlcpBEzWkscBR3Ap1U0Zi\n' +
|
||||
'4oat8wrPZGCblaA6rNkRUVbc/+Vw0stnuJ/BLHbPxyBs6w495yBSjBqUWZMvljNz\n' +
|
||||
'm9/aK0ECgYEA9oVIVAd0enjSVIyAZNbw11ElidzdtBkeIJdsxqhmXzeIFZbB39Gd\n' +
|
||||
'bjtAVclVbq5mLsI1j22ER2rHA4Ygkn6vlLghK3ZMPxZa57oJtmL3oP0RvOjE4zRV\n' +
|
||||
'dzKexNGo9gU/x9SQbuyOmuauvAYhXZxeLpv+lEfsZTqqrvPUGeBiEQcCgYEA8poG\n' +
|
||||
'WVnykWuTmCe0bMmvYDsWpAEiZnFLDaKcSbz3O7RMGbPy1cypmqSinIYUpURBT/WY\n' +
|
||||
'wVPAGtjkuTXtd1Cy58m7PqziB7NNWMcsMGj+lWrTPZ6hCHIBcAImKEPpd+Y9vGJX\n' +
|
||||
'oatFJguqAGOz7rigBq6iPfeQOCWpmprNAuah++cCgYB1gcybOT59TnA7mwlsh8Qf\n' +
|
||||
'bm+tSllnin2A3Y0dGJJLmsXEPKtHS7x2Gcot2h1d98V/TlWHe5WNEUmx1VJbYgXB\n' +
|
||||
'pw8wj2ACxl4ojNYqWPxegaLd4DpRbtW6Tqe9e47FTnU7hIggR6QmFAWAXI+09l8y\n' +
|
||||
'amssNShqjE9lu5YDi6BTKwKBgQCuIlKGViLfsKjrYSyHnajNWPxiUhIgGBf4PI0T\n' +
|
||||
'/Jg1ea/aDykxv0rKHnw9/5vYGIsM2st/kR7l5mMecg/2Qa145HsLfMptHo1ZOPWF\n' +
|
||||
'9gcuttPTegY6aqKPhGthIYX2MwSDMM+X0ri6m0q2JtqjclAjG7yG4CjbtGTt/UlE\n' +
|
||||
'WMlSZwKBgQDslGeLUnkW0bsV5EG3AKRUyPKz/6DVNuxaIRRhOeWVKV101claqXAT\n' +
|
||||
'wXOpdKrvkjZbT4AzcNrlGtRl3l7dEVXTu+dN7/ZieJRu7zaStlAQZkIyP9O3DdQ3\n' +
|
||||
'rIcetQpfrJ1cAqz6Ng0pD0mh77vQ13WG1BBmDFa2A9BuzLoBituf4g==\n' +
|
||||
'-----END RSA PRIVATE KEY-----',
|
||||
cert: '-----BEGIN CERTIFICATE-----\n' +
|
||||
'MIICpDCCAYwCCQCuVLVKVTXnAjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwls\n' +
|
||||
'b2NhbGhvc3QwHhcNMTUwMjEyMTEzMjU4WhcNMjUwMjA5MTEzMjU4WjAUMRIwEAYD\n' +
|
||||
'VQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDp\n' +
|
||||
'nlCqHD6hZ+G0SIwcTfYe33ibBMGkB+O3e8+lfwzsMwJPAezXFxe9DiVDevCt4NM5\n' +
|
||||
'Z2gl4QTLuAzFeofSPDRZ8HH7wgglTr3Gy07JPsVEUO8DXJNFdjbK30rmzpIqR6f9\n' +
|
||||
'5sx+ZlH7Tc0PhQWwPOTPQV+ByOuReaswXkSHlRdf/71gd5TGWnSAwC379K6ZKaAJ\n' +
|
||||
'rKzFkuMul3i5fP5f4lXGyTUAKDU2gQ7uVYkL/dYE/swhy7bBWwx3Zd6AtPSBZh9P\n' +
|
||||
'X+OwgfrhT+ZL9JVcL1CHEyX1h+SjaTHU6JiEPGEezvlElmGs1HGB+LCBJvnQbDg0\n' +
|
||||
'BBoMjGv8Mk90uGCeljpRAgMBAAEwDQYJKoZIhvcNAQELBQADggEBABXm8GPdY0sc\n' +
|
||||
'mMUFlgDqFzcevjdGDce0QfboR+M7WDdm512Jz2SbRTgZD/4na42ThODOZz9z1AcM\n' +
|
||||
'zLgx2ZNZzVhBz0odCU4JVhOCEks/OzSyKeGwjIb4JAY7dh+Kju1+6MNfQJ4r1Hza\n' +
|
||||
'SVXH0+JlpJDaJ73NQ2JyfqELmJ1mTcptkA/N6rQWhlzycTBSlfogwf9xawgVPATP\n' +
|
||||
'4AuwgjHl12JI2HVVs1gu65Y3slvaHRCr0B4+Kg1GYNLLcbFcK+NEHrHmPxy9TnTh\n' +
|
||||
'Zwp1dsNQU+Xkylz8IUANWSLHYZOMtN2e5SKIdwTtl5C8YxveuY8YKb1gDExnMraT\n' +
|
||||
'VGXQDqPleug=\n' +
|
||||
'-----END CERTIFICATE-----'
|
||||
};
|
||||
|
||||
describe('fetch tests', function () {
|
||||
this.timeout(10000); // eslint-disable-line
|
||||
var httpServer, httpsServer;
|
||||
|
||||
beforeEach(function (done) {
|
||||
httpServer = http.createServer(function (req, res) {
|
||||
switch (req.url) {
|
||||
|
||||
case '/redirect6':
|
||||
res.writeHead(302, {
|
||||
Location: '/redirect5'
|
||||
});
|
||||
res.end();
|
||||
break;
|
||||
|
||||
case '/redirect5':
|
||||
res.writeHead(302, {
|
||||
Location: '/redirect4'
|
||||
});
|
||||
res.end();
|
||||
break;
|
||||
|
||||
case '/redirect4':
|
||||
res.writeHead(302, {
|
||||
Location: '/redirect3'
|
||||
});
|
||||
res.end();
|
||||
break;
|
||||
|
||||
case '/redirect3':
|
||||
res.writeHead(302, {
|
||||
Location: '/redirect2'
|
||||
});
|
||||
res.end();
|
||||
break;
|
||||
|
||||
case '/redirect2':
|
||||
res.writeHead(302, {
|
||||
Location: '/redirect1'
|
||||
});
|
||||
res.end();
|
||||
break;
|
||||
|
||||
case '/redirect1':
|
||||
res.writeHead(302, {
|
||||
Location: '/'
|
||||
});
|
||||
res.end();
|
||||
break;
|
||||
|
||||
case '/gzip':
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/plain',
|
||||
'Content-Encoding': 'gzip'
|
||||
});
|
||||
var str = 'H4sIAAAAAAAAA/NIzcnJVwjPL8pJUfAICQngAgCwsOrsEQAAAA==';
|
||||
var strSize = Buffer.byteLength(str, 'base64');
|
||||
var strBuf = USE_ALLOC ? Buffer.alloc(strSize, str, 'base64') : new Buffer(str, 'base64');
|
||||
res.end(strBuf);
|
||||
break;
|
||||
|
||||
case '/invalid':
|
||||
res.writeHead(500, {
|
||||
'Content-Type': 'text/plain'
|
||||
});
|
||||
res.end('Hello World HTTP\n');
|
||||
break;
|
||||
|
||||
case '/auth':
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/plain'
|
||||
});
|
||||
var auth = req.headers.authorization.split(' ').pop();
|
||||
var authSize = Buffer.byteLength(auth, 'base64');
|
||||
var authBuf = USE_ALLOC ? Buffer.alloc(authSize, auth, 'base64') : new Buffer(auth, 'base64');
|
||||
res.end(authBuf);
|
||||
break;
|
||||
|
||||
default:
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/plain'
|
||||
});
|
||||
res.end('Hello World HTTP\n');
|
||||
}
|
||||
});
|
||||
|
||||
httpsServer = https.createServer(httpsOptions, function (req, res) {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/plain'
|
||||
});
|
||||
res.end('Hello World HTTPS\n');
|
||||
});
|
||||
|
||||
httpServer.listen(HTTP_PORT, function () {
|
||||
httpsServer.listen(HTTPS_PORT, done);
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(function (done) {
|
||||
httpServer.close(function () {
|
||||
httpsServer.close(done);
|
||||
});
|
||||
});
|
||||
|
||||
it('should fetch HTTP data', function (done) {
|
||||
var req = new fetch.FetchStream('http://localhost:' + HTTP_PORT);
|
||||
var buf = [];
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('end', function () {
|
||||
expect(Buffer.concat(buf).toString()).to.equal('Hello World HTTP\n');
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should fetch HTTPS data', function (done) {
|
||||
var req = new fetch.FetchStream('https://localhost:' + HTTPS_PORT, {
|
||||
rejectUnauthorized: false
|
||||
});
|
||||
var buf = [];
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('end', function () {
|
||||
expect(Buffer.concat(buf).toString()).to.equal('Hello World HTTPS\n');
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should fail on self signed HTTPS certificate', function (done) {
|
||||
var req = new fetch.FetchStream('https://localhost:' + HTTPS_PORT);
|
||||
req.on('error', function (err) {
|
||||
expect(err).to.exist;
|
||||
done();
|
||||
});
|
||||
req.on('data', function () {
|
||||
expect(false).to.be.true;
|
||||
});
|
||||
req.on('end', function () {
|
||||
expect(false).to.be.true;
|
||||
});
|
||||
});
|
||||
|
||||
it('should fetch HTTP data with redirects', function (done) {
|
||||
var req = new fetch.FetchStream('http://localhost:' + HTTP_PORT + '/redirect3');
|
||||
var buf = [];
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('end', function () {
|
||||
expect(Buffer.concat(buf).toString()).to.equal('Hello World HTTP\n');
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should not follow too many redirects', function (done) {
|
||||
var req = new fetch.FetchStream('http://localhost:' + HTTP_PORT + '/redirect6', {
|
||||
maxRedirects: 5
|
||||
});
|
||||
|
||||
req.on('meta', function (meta) {
|
||||
expect(meta.status).to.equal(302);
|
||||
});
|
||||
|
||||
var buf = [];
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('end', function () {
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should unzip compressed HTTP data', function (done) {
|
||||
var req = new fetch.FetchStream('http://localhost:' + HTTP_PORT + '/gzip');
|
||||
var buf = [];
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('end', function () {
|
||||
expect(Buffer.concat(buf).toString()).to.equal('Hello World HTTP\n');
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should return error for unresolved host', function (done) {
|
||||
var req = new fetch.FetchStream('http://asfhaskhhgbjdsfhgbsdjgk');
|
||||
var buf = [];
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('error', function (err) {
|
||||
expect(err).to.exist;
|
||||
done();
|
||||
});
|
||||
req.on('end', function () {});
|
||||
});
|
||||
|
||||
it('should handle basic HTTP auth', function (done) {
|
||||
var req = new fetch.FetchStream('http://user:pass@localhost:' + HTTP_PORT + '/auth');
|
||||
var buf = [];
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('end', function () {
|
||||
expect(Buffer.concat(buf).toString()).to.equal('user:pass');
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should return error for invalid protocol', function (done) {
|
||||
var req = new fetch.FetchStream('http://localhost:' + HTTPS_PORT, {
|
||||
timeout: 1000
|
||||
});
|
||||
var buf = [];
|
||||
|
||||
req.on('data', function (chunk) {
|
||||
buf.push(chunk);
|
||||
});
|
||||
req.on('error', function (err) {
|
||||
expect(err).to.exist;
|
||||
done();
|
||||
});
|
||||
req.on('end', function () {});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user