nodejs write 64bit unsigned integer to buffer

Question

I want to store a 64bit (8 byte) big integer to a nodejs buffer object in big endian format.

The problem about this task is that nodejs buffer only supports writing 32bit integers as maximum (with buf.write32UInt32BE(value, offset)). So I thought, why can't we just split the 64bit integer?

var buf = new Buffer(8);

buf.fill(0) // clear all bytes of the buffer
console.log(buf); // outputs <Buffer 00 00 00 00 00 00 00 00>

var int = 0xffff; // as dezimal: 65535

buf.write32UInt32BE(0xff, 4); // right the first part of the int
console.log(buf); // outputs <Buffer 00 00 00 00 00 00 00 ff>

buf.write32UInt32BE(0xff, 0); // right the second part of the int
console.log(buf); // outputs <Buffer 00 00 00 ff 00 00 00 ff>

var bufInt = buf.read32UInt32BE(0) * buf.read32UInt32BE(4);
console.log(bufInt); // outputs 65025

As you see this nearly works. The problem is just splitting the 64bit integer and finding the missing 510 at reading it. Would somebody mind showing the solutions for these two issues?

Solution 1

I think what you are looking for is:

var bufInt = (buf.readUInt32BE(0) << 8) + buf.readUInt32BE(4);

Shift the first number by 8 bits and add (instead of multiplying), wich returns 65535

---

EDIT

Another way to write would be:

var buf = new Buffer(8);
buf.fill(0);

var i = 0xCDEF; // 52719 in decimal

buf.writeUInt32BE(i >> 8, 0); //write the high order bits (shifted over)
buf.writeUInt32BE(i & 0x00ff, 4); //write the low order bits

console.log(buf); //displays: <Buffer 00 00 00 cd 00 00 00 ef>

var bufInt = (buf.readUInt32BE(0) << 8) + buf.readUInt32BE(4);
console.log(bufInt); //displays: 52719

Solution 2

I'm confused because your example value of 0xFFFF is only 16-bit, not 64-bit.

Keep in mind that the JS number type is specified as an IEEE754 floating-point value, so it is not guaranteed to be able to hold a 64-bit unsigned value. If you want real 64-bit integer support, you need to use a module to provide it, like bignum. The readme for that has examples of reading and writing values to buffers.

Floating-point values can only represent values up to 2^53 - 1 without losing precision. You can see that in this example using standard JS numbers:

var b = new Buffer([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF])
var firstHalf = b.readUInt32BE(0); // 4294967295
var secondHalf = b.readUInt32BE(4); // 4294967295

var val = firstHalf * 0x100000000 + secondHalf; // 18446744073709552000

The result of this is 18446744073709552000 when the proper value is 18446744073709551615.

var bignum = require('bignum');

var b = new Buffer([0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF])
var val = bignum.fromBuffer(b);

This results in a BigNum object with the value 18446744073709551615.

Also, to elaborate on your example code, the value you are using is only 16-bit, and you are trying to work with it using 32-bit functions. You can just do this:

var buf = new Buffer(2);

buf.fill(0) // clear all bytes of the buffer
console.log(buf); // outputs <Buffer 00 00>

var int = 0xffff; // as decimal: 65535

// Write it with a standard 16-bit function calls.
buf.writeUInt16BE(int);

// OR write it with 2 8-bit function calls.
buf.writeUInt8(int & 0xff, 0); // right the first part of the int
buf.writeUInt8((int >> 8) & 0xFF, 1); // right the second part of the int
console.log(buf); // outputs <Buffer ff ff>

// Read it as a 16-bit value.
var bufInt = buf.readUInt16BE(0);
console.log(bufInt);

// OR read it as two 8-bit values.
var bufInt = (buf.readUInt8(1) << 8) + buf.readUInt8(0);

Solution 3

Reading and writings UINT numbers up to Number.MAX_SAFE_INTEGER.

This works in node.js only, is not portable on browser side.

function uintToBase62(n) {
  if (n < 0) throw 'unsupported negative integer';

  let uintBuffer;
  if (n < 0x7FFFFFFF) {
    uintBuffer = new Buffer(4);
    uintBuffer.writeUInt32BE(n, 0);
  } else {
    // `~~` double bitwise operator
    // The most practical way of utilizing the power of this operator is to use it as a replacement
    // for Math.floor() function as double bitwise NOT performs the same operation a lot quicker.
    // You can use it, to convert any floating point number to a integer without performance overkill
    // that comes with Math.floor(). Additionally, when you care about minification of your code,
    // you end up using 2 characters (2 tildes) instead of 12.
    // http://rocha.la/JavaScript-bitwise-operators-in-practice
    const big = ~~(n / 0x0100000000);
    const low = (n % 0x0100000000);
    uintBuffer = new Buffer(8);
    uintBuffer.writeUInt32BE(big, 0);
    uintBuffer.writeUInt32BE(low, 4);
  }

  return uintBuffer.toString('hex');
}

to convert it

function uintFromBase62(uintBuffer) {
  const n = parseInt(uintBuffer.toString('hex'), 16);
  return n;
}