| "use strict"; |
| var Buffer = require("safer-buffer").Buffer; |
| |
| // Multibyte codec. In this scheme, a character is represented by 1 or more bytes. |
| // Our codec supports UTF-16 surrogates, extensions for GB18030 and unicode sequences. |
| // To save memory and loading time, we read table files only when requested. |
| |
| exports._dbcs = DBCSCodec; |
| |
| var UNASSIGNED = -1, |
| GB18030_CODE = -2, |
| SEQ_START = -10, |
| NODE_START = -1000, |
| UNASSIGNED_NODE = new Array(0x100), |
| DEF_CHAR = -1; |
| |
| for (var i = 0; i < 0x100; i++) |
| UNASSIGNED_NODE[i] = UNASSIGNED; |
| |
| |
| // Class DBCSCodec reads and initializes mapping tables. |
| function DBCSCodec(codecOptions, iconv) { |
| this.encodingName = codecOptions.encodingName; |
| if (!codecOptions) |
| throw new Error("DBCS codec is called without the data.") |
| if (!codecOptions.table) |
| throw new Error("Encoding '" + this.encodingName + "' has no data."); |
| |
| // Load tables. |
| var mappingTable = codecOptions.table(); |
| |
| |
| // Decode tables: MBCS -> Unicode. |
| |
| // decodeTables is a trie, encoded as an array of arrays of integers. Internal arrays are trie nodes and all have len = 256. |
| // Trie root is decodeTables[0]. |
| // Values: >= 0 -> unicode character code. can be > 0xFFFF |
| // == UNASSIGNED -> unknown/unassigned sequence. |
| // == GB18030_CODE -> this is the end of a GB18030 4-byte sequence. |
| // <= NODE_START -> index of the next node in our trie to process next byte. |
| // <= SEQ_START -> index of the start of a character code sequence, in decodeTableSeq. |
| this.decodeTables = []; |
| this.decodeTables[0] = UNASSIGNED_NODE.slice(0); // Create root node. |
| |
| // Sometimes a MBCS char corresponds to a sequence of unicode chars. We store them as arrays of integers here. |
| this.decodeTableSeq = []; |
| |
| // Actual mapping tables consist of chunks. Use them to fill up decode tables. |
| for (var i = 0; i < mappingTable.length; i++) |
| this._addDecodeChunk(mappingTable[i]); |
| |
| this.defaultCharUnicode = iconv.defaultCharUnicode; |
| |
| |
| // Encode tables: Unicode -> DBCS. |
| |
| // `encodeTable` is array mapping from unicode char to encoded char. All its values are integers for performance. |
| // Because it can be sparse, it is represented as array of buckets by 256 chars each. Bucket can be null. |
| // Values: >= 0 -> it is a normal char. Write the value (if <=256 then 1 byte, if <=65536 then 2 bytes, etc.). |
| // == UNASSIGNED -> no conversion found. Output a default char. |
| // <= SEQ_START -> it's an index in encodeTableSeq, see below. The character starts a sequence. |
| this.encodeTable = []; |
| |
| // `encodeTableSeq` is used when a sequence of unicode characters is encoded as a single code. We use a tree of |
| // objects where keys correspond to characters in sequence and leafs are the encoded dbcs values. A special DEF_CHAR key |
| // means end of sequence (needed when one sequence is a strict subsequence of another). |
| // Objects are kept separately from encodeTable to increase performance. |
| this.encodeTableSeq = []; |
| |
| // Some chars can be decoded, but need not be encoded. |
| var skipEncodeChars = {}; |
| if (codecOptions.encodeSkipVals) |
| for (var i = 0; i < codecOptions.encodeSkipVals.length; i++) { |
| var val = codecOptions.encodeSkipVals[i]; |
| if (typeof val === 'number') |
| skipEncodeChars[val] = true; |
| else |
| for (var j = val.from; j <= val.to; j++) |
| skipEncodeChars[j] = true; |
| } |
| |
| // Use decode trie to recursively fill out encode tables. |
| this._fillEncodeTable(0, 0, skipEncodeChars); |
| |
| // Add more encoding pairs when needed. |
| if (codecOptions.encodeAdd) { |
| for (var uChar in codecOptions.encodeAdd) |
| if (Object.prototype.hasOwnProperty.call(codecOptions.encodeAdd, uChar)) |
| this._setEncodeChar(uChar.charCodeAt(0), codecOptions.encodeAdd[uChar]); |
| } |
| |
| this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)]; |
| if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?']; |
| if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0); |
| |
| |
| // Load & create GB18030 tables when needed. |
| if (typeof codecOptions.gb18030 === 'function') { |
| this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges. |
| |
| // Add GB18030 decode tables. |
| var thirdByteNodeIdx = this.decodeTables.length; |
| var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0); |
| |
| var fourthByteNodeIdx = this.decodeTables.length; |
| var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0); |
| |
| for (var i = 0x81; i <= 0xFE; i++) { |
| var secondByteNodeIdx = NODE_START - this.decodeTables[0][i]; |
| var secondByteNode = this.decodeTables[secondByteNodeIdx]; |
| for (var j = 0x30; j <= 0x39; j++) |
| secondByteNode[j] = NODE_START - thirdByteNodeIdx; |
| } |
| for (var i = 0x81; i <= 0xFE; i++) |
| thirdByteNode[i] = NODE_START - fourthByteNodeIdx; |
| for (var i = 0x30; i <= 0x39; i++) |
| fourthByteNode[i] = GB18030_CODE |
| } |
| } |
| |
| DBCSCodec.prototype.encoder = DBCSEncoder; |
| DBCSCodec.prototype.decoder = DBCSDecoder; |
| |
| // Decoder helpers |
| DBCSCodec.prototype._getDecodeTrieNode = function(addr) { |
| var bytes = []; |
| for (; addr > 0; addr >>= 8) |
| bytes.push(addr & 0xFF); |
| if (bytes.length == 0) |
| bytes.push(0); |
| |
| var node = this.decodeTables[0]; |
| for (var i = bytes.length-1; i > 0; i--) { // Traverse nodes deeper into the trie. |
| var val = node[bytes[i]]; |
| |
| if (val == UNASSIGNED) { // Create new node. |
| node[bytes[i]] = NODE_START - this.decodeTables.length; |
| this.decodeTables.push(node = UNASSIGNED_NODE.slice(0)); |
| } |
| else if (val <= NODE_START) { // Existing node. |
| node = this.decodeTables[NODE_START - val]; |
| } |
| else |
| throw new Error("Overwrite byte in " + this.encodingName + ", addr: " + addr.toString(16)); |
| } |
| return node; |
| } |
| |
| |
| DBCSCodec.prototype._addDecodeChunk = function(chunk) { |
| // First element of chunk is the hex mbcs code where we start. |
| var curAddr = parseInt(chunk[0], 16); |
| |
| // Choose the decoding node where we'll write our chars. |
| var writeTable = this._getDecodeTrieNode(curAddr); |
| curAddr = curAddr & 0xFF; |
| |
| // Write all other elements of the chunk to the table. |
| for (var k = 1; k < chunk.length; k++) { |
| var part = chunk[k]; |
| if (typeof part === "string") { // String, write as-is. |
| for (var l = 0; l < part.length;) { |
| var code = part.charCodeAt(l++); |
| if (0xD800 <= code && code < 0xDC00) { // Decode surrogate |
| var codeTrail = part.charCodeAt(l++); |
| if (0xDC00 <= codeTrail && codeTrail < 0xE000) |
| writeTable[curAddr++] = 0x10000 + (code - 0xD800) * 0x400 + (codeTrail - 0xDC00); |
| else |
| throw new Error("Incorrect surrogate pair in " + this.encodingName + " at chunk " + chunk[0]); |
| } |
| else if (0x0FF0 < code && code <= 0x0FFF) { // Character sequence (our own encoding used) |
| var len = 0xFFF - code + 2; |
| var seq = []; |
| for (var m = 0; m < len; m++) |
| seq.push(part.charCodeAt(l++)); // Simple variation: don't support surrogates or subsequences in seq. |
| |
| writeTable[curAddr++] = SEQ_START - this.decodeTableSeq.length; |
| this.decodeTableSeq.push(seq); |
| } |
| else |
| writeTable[curAddr++] = code; // Basic char |
| } |
| } |
| else if (typeof part === "number") { // Integer, meaning increasing sequence starting with prev character. |
| var charCode = writeTable[curAddr - 1] + 1; |
| for (var l = 0; l < part; l++) |
| writeTable[curAddr++] = charCode++; |
| } |
| else |
| throw new Error("Incorrect type '" + typeof part + "' given in " + this.encodingName + " at chunk " + chunk[0]); |
| } |
| if (curAddr > 0xFF) |
| throw new Error("Incorrect chunk in " + this.encodingName + " at addr " + chunk[0] + ": too long" + curAddr); |
| } |
| |
| // Encoder helpers |
| DBCSCodec.prototype._getEncodeBucket = function(uCode) { |
| var high = uCode >> 8; // This could be > 0xFF because of astral characters. |
| if (this.encodeTable[high] === undefined) |
| this.encodeTable[high] = UNASSIGNED_NODE.slice(0); // Create bucket on demand. |
| return this.encodeTable[high]; |
| } |
| |
| DBCSCodec.prototype._setEncodeChar = function(uCode, dbcsCode) { |
| var bucket = this._getEncodeBucket(uCode); |
| var low = uCode & 0xFF; |
| if (bucket[low] <= SEQ_START) |
| this.encodeTableSeq[SEQ_START-bucket[low]][DEF_CHAR] = dbcsCode; // There's already a sequence, set a single-char subsequence of it. |
| else if (bucket[low] == UNASSIGNED) |
| bucket[low] = dbcsCode; |
| } |
| |
| DBCSCodec.prototype._setEncodeSequence = function(seq, dbcsCode) { |
| |
| // Get the root of character tree according to first character of the sequence. |
| var uCode = seq[0]; |
| var bucket = this._getEncodeBucket(uCode); |
| var low = uCode & 0xFF; |
| |
| var node; |
| if (bucket[low] <= SEQ_START) { |
| // There's already a sequence with - use it. |
| node = this.encodeTableSeq[SEQ_START-bucket[low]]; |
| } |
| else { |
| // There was no sequence object - allocate a new one. |
| node = {}; |
| if (bucket[low] !== UNASSIGNED) node[DEF_CHAR] = bucket[low]; // If a char was set before - make it a single-char subsequence. |
| bucket[low] = SEQ_START - this.encodeTableSeq.length; |
| this.encodeTableSeq.push(node); |
| } |
| |
| // Traverse the character tree, allocating new nodes as needed. |
| for (var j = 1; j < seq.length-1; j++) { |
| var oldVal = node[uCode]; |
| if (typeof oldVal === 'object') |
| node = oldVal; |
| else { |
| node = node[uCode] = {} |
| if (oldVal !== undefined) |
| node[DEF_CHAR] = oldVal |
| } |
| } |
| |
| // Set the leaf to given dbcsCode. |
| uCode = seq[seq.length-1]; |
| node[uCode] = dbcsCode; |
| } |
| |
| DBCSCodec.prototype._fillEncodeTable = function(nodeIdx, prefix, skipEncodeChars) { |
| var node = this.decodeTables[nodeIdx]; |
| for (var i = 0; i < 0x100; i++) { |
| var uCode = node[i]; |
| var mbCode = prefix + i; |
| if (skipEncodeChars[mbCode]) |
| continue; |
| |
| if (uCode >= 0) |
| this._setEncodeChar(uCode, mbCode); |
| else if (uCode <= NODE_START) |
| this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars); |
| else if (uCode <= SEQ_START) |
| this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode); |
| } |
| } |
| |
| |
| |
| // == Encoder ================================================================== |
| |
| function DBCSEncoder(options, codec) { |
| // Encoder state |
| this.leadSurrogate = -1; |
| this.seqObj = undefined; |
| |
| // Static data |
| this.encodeTable = codec.encodeTable; |
| this.encodeTableSeq = codec.encodeTableSeq; |
| this.defaultCharSingleByte = codec.defCharSB; |
| this.gb18030 = codec.gb18030; |
| } |
| |
| DBCSEncoder.prototype.write = function(str) { |
| var newBuf = Buffer.alloc(str.length * (this.gb18030 ? 4 : 3)), |
| leadSurrogate = this.leadSurrogate, |
| seqObj = this.seqObj, nextChar = -1, |
| i = 0, j = 0; |
| |
| while (true) { |
| // 0. Get next character. |
| if (nextChar === -1) { |
| if (i == str.length) break; |
| var uCode = str.charCodeAt(i++); |
| } |
| else { |
| var uCode = nextChar; |
| nextChar = -1; |
| } |
| |
| // 1. Handle surrogates. |
| if (0xD800 <= uCode && uCode < 0xE000) { // Char is one of surrogates. |
| if (uCode < 0xDC00) { // We've got lead surrogate. |
| if (leadSurrogate === -1) { |
| leadSurrogate = uCode; |
| continue; |
| } else { |
| leadSurrogate = uCode; |
| // Double lead surrogate found. |
| uCode = UNASSIGNED; |
| } |
| } else { // We've got trail surrogate. |
| if (leadSurrogate !== -1) { |
| uCode = 0x10000 + (leadSurrogate - 0xD800) * 0x400 + (uCode - 0xDC00); |
| leadSurrogate = -1; |
| } else { |
| // Incomplete surrogate pair - only trail surrogate found. |
| uCode = UNASSIGNED; |
| } |
| |
| } |
| } |
| else if (leadSurrogate !== -1) { |
| // Incomplete surrogate pair - only lead surrogate found. |
| nextChar = uCode; uCode = UNASSIGNED; // Write an error, then current char. |
| leadSurrogate = -1; |
| } |
| |
| // 2. Convert uCode character. |
| var dbcsCode = UNASSIGNED; |
| if (seqObj !== undefined && uCode != UNASSIGNED) { // We are in the middle of the sequence |
| var resCode = seqObj[uCode]; |
| if (typeof resCode === 'object') { // Sequence continues. |
| seqObj = resCode; |
| continue; |
| |
| } else if (typeof resCode == 'number') { // Sequence finished. Write it. |
| dbcsCode = resCode; |
| |
| } else if (resCode == undefined) { // Current character is not part of the sequence. |
| |
| // Try default character for this sequence |
| resCode = seqObj[DEF_CHAR]; |
| if (resCode !== undefined) { |
| dbcsCode = resCode; // Found. Write it. |
| nextChar = uCode; // Current character will be written too in the next iteration. |
| |
| } else { |
| // TODO: What if we have no default? (resCode == undefined) |
| // Then, we should write first char of the sequence as-is and try the rest recursively. |
| // Didn't do it for now because no encoding has this situation yet. |
| // Currently, just skip the sequence and write current char. |
| } |
| } |
| seqObj = undefined; |
| } |
| else if (uCode >= 0) { // Regular character |
| var subtable = this.encodeTable[uCode >> 8]; |
| if (subtable !== undefined) |
| dbcsCode = subtable[uCode & 0xFF]; |
| |
| if (dbcsCode <= SEQ_START) { // Sequence start |
| seqObj = this.encodeTableSeq[SEQ_START-dbcsCode]; |
| continue; |
| } |
| |
| if (dbcsCode == UNASSIGNED && this.gb18030) { |
| // Use GB18030 algorithm to find character(s) to write. |
| var idx = findIdx(this.gb18030.uChars, uCode); |
| if (idx != -1) { |
| var dbcsCode = this.gb18030.gbChars[idx] + (uCode - this.gb18030.uChars[idx]); |
| newBuf[j++] = 0x81 + Math.floor(dbcsCode / 12600); dbcsCode = dbcsCode % 12600; |
| newBuf[j++] = 0x30 + Math.floor(dbcsCode / 1260); dbcsCode = dbcsCode % 1260; |
| newBuf[j++] = 0x81 + Math.floor(dbcsCode / 10); dbcsCode = dbcsCode % 10; |
| newBuf[j++] = 0x30 + dbcsCode; |
| continue; |
| } |
| } |
| } |
| |
| // 3. Write dbcsCode character. |
| if (dbcsCode === UNASSIGNED) |
| dbcsCode = this.defaultCharSingleByte; |
| |
| if (dbcsCode < 0x100) { |
| newBuf[j++] = dbcsCode; |
| } |
| else if (dbcsCode < 0x10000) { |
| newBuf[j++] = dbcsCode >> 8; // high byte |
| newBuf[j++] = dbcsCode & 0xFF; // low byte |
| } |
| else { |
| newBuf[j++] = dbcsCode >> 16; |
| newBuf[j++] = (dbcsCode >> 8) & 0xFF; |
| newBuf[j++] = dbcsCode & 0xFF; |
| } |
| } |
| |
| this.seqObj = seqObj; |
| this.leadSurrogate = leadSurrogate; |
| return newBuf.slice(0, j); |
| } |
| |
| DBCSEncoder.prototype.end = function() { |
| if (this.leadSurrogate === -1 && this.seqObj === undefined) |
| return; // All clean. Most often case. |
| |
| var newBuf = Buffer.alloc(10), j = 0; |
| |
| if (this.seqObj) { // We're in the sequence. |
| var dbcsCode = this.seqObj[DEF_CHAR]; |
| if (dbcsCode !== undefined) { // Write beginning of the sequence. |
| if (dbcsCode < 0x100) { |
| newBuf[j++] = dbcsCode; |
| } |
| else { |
| newBuf[j++] = dbcsCode >> 8; // high byte |
| newBuf[j++] = dbcsCode & 0xFF; // low byte |
| } |
| } else { |
| // See todo above. |
| } |
| this.seqObj = undefined; |
| } |
| |
| if (this.leadSurrogate !== -1) { |
| // Incomplete surrogate pair - only lead surrogate found. |
| newBuf[j++] = this.defaultCharSingleByte; |
| this.leadSurrogate = -1; |
| } |
| |
| return newBuf.slice(0, j); |
| } |
| |
| // Export for testing |
| DBCSEncoder.prototype.findIdx = findIdx; |
| |
| |
| // == Decoder ================================================================== |
| |
| function DBCSDecoder(options, codec) { |
| // Decoder state |
| this.nodeIdx = 0; |
| this.prevBuf = Buffer.alloc(0); |
| |
| // Static data |
| this.decodeTables = codec.decodeTables; |
| this.decodeTableSeq = codec.decodeTableSeq; |
| this.defaultCharUnicode = codec.defaultCharUnicode; |
| this.gb18030 = codec.gb18030; |
| } |
| |
| DBCSDecoder.prototype.write = function(buf) { |
| var newBuf = Buffer.alloc(buf.length*2), |
| nodeIdx = this.nodeIdx, |
| prevBuf = this.prevBuf, prevBufOffset = this.prevBuf.length, |
| seqStart = -this.prevBuf.length, // idx of the start of current parsed sequence. |
| uCode; |
| |
| if (prevBufOffset > 0) // Make prev buf overlap a little to make it easier to slice later. |
| prevBuf = Buffer.concat([prevBuf, buf.slice(0, 10)]); |
| |
| for (var i = 0, j = 0; i < buf.length; i++) { |
| var curByte = (i >= 0) ? buf[i] : prevBuf[i + prevBufOffset]; |
| |
| // Lookup in current trie node. |
| var uCode = this.decodeTables[nodeIdx][curByte]; |
| |
| if (uCode >= 0) { |
| // Normal character, just use it. |
| } |
| else if (uCode === UNASSIGNED) { // Unknown char. |
| // TODO: Callback with seq. |
| //var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset); |
| i = seqStart; // Try to parse again, after skipping first byte of the sequence ('i' will be incremented by 'for' cycle). |
| uCode = this.defaultCharUnicode.charCodeAt(0); |
| } |
| else if (uCode === GB18030_CODE) { |
| var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset); |
| var ptr = (curSeq[0]-0x81)*12600 + (curSeq[1]-0x30)*1260 + (curSeq[2]-0x81)*10 + (curSeq[3]-0x30); |
| var idx = findIdx(this.gb18030.gbChars, ptr); |
| uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx]; |
| } |
| else if (uCode <= NODE_START) { // Go to next trie node. |
| nodeIdx = NODE_START - uCode; |
| continue; |
| } |
| else if (uCode <= SEQ_START) { // Output a sequence of chars. |
| var seq = this.decodeTableSeq[SEQ_START - uCode]; |
| for (var k = 0; k < seq.length - 1; k++) { |
| uCode = seq[k]; |
| newBuf[j++] = uCode & 0xFF; |
| newBuf[j++] = uCode >> 8; |
| } |
| uCode = seq[seq.length-1]; |
| } |
| else |
| throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte); |
| |
| // Write the character to buffer, handling higher planes using surrogate pair. |
| if (uCode > 0xFFFF) { |
| uCode -= 0x10000; |
| var uCodeLead = 0xD800 + Math.floor(uCode / 0x400); |
| newBuf[j++] = uCodeLead & 0xFF; |
| newBuf[j++] = uCodeLead >> 8; |
| |
| uCode = 0xDC00 + uCode % 0x400; |
| } |
| newBuf[j++] = uCode & 0xFF; |
| newBuf[j++] = uCode >> 8; |
| |
| // Reset trie node. |
| nodeIdx = 0; seqStart = i+1; |
| } |
| |
| this.nodeIdx = nodeIdx; |
| this.prevBuf = (seqStart >= 0) ? buf.slice(seqStart) : prevBuf.slice(seqStart + prevBufOffset); |
| return newBuf.slice(0, j).toString('ucs2'); |
| } |
| |
| DBCSDecoder.prototype.end = function() { |
| var ret = ''; |
| |
| // Try to parse all remaining chars. |
| while (this.prevBuf.length > 0) { |
| // Skip 1 character in the buffer. |
| ret += this.defaultCharUnicode; |
| var buf = this.prevBuf.slice(1); |
| |
| // Parse remaining as usual. |
| this.prevBuf = Buffer.alloc(0); |
| this.nodeIdx = 0; |
| if (buf.length > 0) |
| ret += this.write(buf); |
| } |
| |
| this.nodeIdx = 0; |
| return ret; |
| } |
| |
| // Binary search for GB18030. Returns largest i such that table[i] <= val. |
| function findIdx(table, val) { |
| if (table[0] > val) |
| return -1; |
| |
| var l = 0, r = table.length; |
| while (l < r-1) { // always table[l] <= val < table[r] |
| var mid = l + Math.floor((r-l+1)/2); |
| if (table[mid] <= val) |
| l = mid; |
| else |
| r = mid; |
| } |
| return l; |
| } |
| |
