Merge pull request #308 from alexburley/master

Charset detection not default as utf-8 #306

lib/decoder.js

@@ -19,7 +19,7 @@ StreamDecoder.prototype._transform = function(chunk, encoding, done) {
   var res, found;
 
   // try get charset from chunk, just once
-  if (this.charset == 'iso-8859-1' && !this.parsed_chunk) {
+  if (this.charset == 'utf8' && !this.parsed_chunk) {
     this.parsed_chunk = true;
 
     var matches = regex.exec(chunk.toString());

lib/needle.js

@@ -128,7 +128,7 @@ function keys_by_type(type) {
 function parse_content_type(header) {
   if (!header || header === '') return {};
 
-  var found, charset = 'iso-8859-1', arr = header.split(';');
+  var found, charset = 'utf8', arr = header.split(';');
 
   if (arr.length > 1 && (found = arr[1].match(/charset=(.+)/)))
     charset = found[1];
@@ -589,10 +589,9 @@ Needle.prototype.send_request = function(count, method, uri, config, post_data,
     // If we're not parsing, and unless decoding was disabled, we'll try
     // decoding non UTF-8 bodies to UTF-8, using the iconv-lite library.
     } else if (text_response && config.decode_response
-      && mime.charset && !mime.charset.match(/utf-?8$/i)) {
+      && mime.charset) {
         pipeline.push(decoder(mime.charset));
     }
-
     // And `out` is the stream we finally push the decoded/parsed output to.
     pipeline.push(out);
 

package.json

@@ -48,6 +48,7 @@
     "JSONStream": "^1.3.5",
     "jschardet": "^1.6.0",
     "mocha": "^5.2.0",
+    "nock": "^12.0.3",
     "q": "^1.5.1",
     "should": "^13.2.3",
     "sinon": "^2.3.0",

test/decoder_spec.js

@@ -2,13 +2,14 @@ var should  = require('should'),
     needle  = require('./../'),
     Q       = require('q'),
     chardet = require('jschardet');
+    nock    = require('nock')
 
 describe('character encoding', function() {
 
   var url;
   this.timeout(5000);
 
-  describe('test A', function() {
+  describe('Given content-type: "text/html; charset=EUC-JP"', function() {
 
     before(function() {
       url = 'http://www.nina.jp/server/slackware/webapp/tomcat_charset.html';
@@ -46,7 +47,7 @@ describe('character encoding', function() {
 
   })
 
-  describe('test B', function() {
+  describe('Given content-type: "text/html but file is charset: gb2312', function() {
 
     it('encodes to UTF-8', function(done) {
 
@@ -71,16 +72,43 @@ describe('character encoding', function() {
           chardet.detect(bodies[1]).encoding,
         ]
 
-        // We wanted to decode our first stream.
+        // We wanted to decode our first stream as specified by options
         charsets[0].should.equal('ascii');
         bodies[0].indexOf('全球中文网站前二十强').should.not.equal(-1);
 
-        // But not our second stream.
+        // But not our second stream
         charsets[1].should.equal('windows-1252');
         bodies[1].indexOf('全球中文网站前二十强').should.equal(-1);
 
         done();
       });
     })
   })
+
+  describe('Given content-type: "text/html"', function () {
+    var hungarianUrl = 'https://some.domain.com';
+    beforeEach(function () {
+      nock(hungarianUrl)
+        .get('/')
+        .reply(200, 'Magyarországi Fióktelepe', {
+          'content-type': 'text/html',
+        });
+    })
+    describe('with decode = false', function () {
+
+      it('decodes by default to utf-8', function (done) {
+
+        needle.get(hungarianUrl, { decode: false }, function (err, resp) {
+          console.log(resp.body)
+          resp.body.should.be.a.String;
+          chardet.detect(resp.body).encoding.should.eql('ISO-8859-2');
+          resp.body.should.eql('Magyarországi Fióktelepe')
+          done();
+        })
+
+      })
+
+    })
+
+  })
 })