Bit-Based BinaryWriter in C#

Question

I'm working on a bit-based B/W/Greyscale Pre-Compiled font format, and was having issues with either reading or writing the format, (I've not been able to determine where the issue was. (I do have a B/W bit-based version working, but an Aliased font doesn't look too good, as you can imagine, especially when working with a 320x200 pixel screen) ) but decided that just using a BinaryWriter would be much easier than writing to a bool[] when I pulled the image data.

The basic format of a pixel in the file looks like this:

1 - White Pixel (Shortest, as this would be most of the pixels)

00 - Black Pixel (No reason to write 10-bits for a pure black pixel, which there are a reasonable number of)

01 - Greyscale Pixel, and is followed by 1 byte describing the shade of the pixel

Now, everything is fine and dandy with writing the required info, as that's all full bytes, but the default .Net 4.0 BinaryWriter writes a Boolean value as a full byte, and as you can imagine, that negates the use of a bit-based format. So I was wondering, is there a BinaryWriter, (and BinaryReader) implementation out there that's bit-based

Edit: I ended up creating my own. (See the answer for the code.)

Answer 1

I ended up writing my own, so here they are.

The BinaryWriter (I've only overridden the ones that I needed)

private class BinaryWriter : System.IO.BinaryWriter
{
    private bool[] curByte = new bool[8];
    private byte curBitIndx = 0;
    private System.Collections.BitArray ba;

    public BinaryWriter(Stream s) : base(s) { }

    public override void Flush()
    {
        base.Write(ConvertToByte(curByte));
        base.Flush();
    }

    public override void Write(bool value)
    {
        curByte[curBitIndx] = value;
        curBitIndx++;

        if (curBitIndx == 8)
        {
            base.Write(ConvertToByte(curByte));
            this.curBitIndx = 0;
            this.curByte = new bool[8];
        }
    }

    public override void Write(byte value)
    {
        ba = new BitArray(new byte[] { value });
        for (byte i = 0; i < 8; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    public override void Write(byte[] buffer)
    {
        for (int i = 0; i < buffer.Length; i++)
        {
            this.Write((byte)buffer[i]);
        }
    }

    public override void Write(uint value)
    {
        ba = new BitArray(BitConverter.GetBytes(value));
        for (byte i = 0; i < 32; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    public override void Write(ulong value)
    {
        ba = new BitArray(BitConverter.GetBytes(value));
        for (byte i = 0; i < 64; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    public override void Write(ushort value)
    {
        ba = new BitArray(BitConverter.GetBytes(value));
        for (byte i = 0; i < 16; i++)
        {
            this.Write(ba[i]);
        }
        ba = null;
    }

    private static byte ConvertToByte(bool[] bools)
    {
        byte b = 0;

        byte bitIndex = 0;
        for (int i = 0; i < 8; i++)
        {
            if (bools[i])
            {
                b |= (byte)(((byte)1) << bitIndex);
            }
            bitIndex++;
        }

        return b;
    }
}

And, the BinaryReader, once again, I've only overridden the methods that I needed.

private class BinaryReader : System.IO.BinaryReader
{
    private bool[] curByte = new bool[8];
    private byte curBitIndx = 0;
    private BitArray ba;

    public BinaryReader(Stream s) : base(s)
    {
        ba = new BitArray(new byte[] { base.ReadByte() });
        ba.CopyTo(curByte, 0);
        ba = null;
    }

    public override bool ReadBoolean()
    {
        if (curBitIndx == 8)
        {
            ba = new BitArray(new byte[] { base.ReadByte() });
            ba.CopyTo(curByte, 0);
            ba = null;
            this.curBitIndx = 0;
        }

        bool b = curByte[curBitIndx];
        curBitIndx++;
        return b;
    }

    public override byte ReadByte()
    {
        bool[] bar = new bool[8];
        byte i;
        for (i = 0; i < 8; i++)
        {
            bar[i] = this.ReadBoolean();
        }

        byte b = 0;
        byte bitIndex = 0;
        for (i = 0; i < 8; i++)
        {
            if (bar[i])
            {
                b |= (byte)(((byte)1) << bitIndex);
            }
            bitIndex++;
        }
        return b;
    }

    public override byte[] ReadBytes(int count)
    {
        byte[] bytes = new byte[count];
        for (int i = 0; i < count; i++)
        {
            bytes[i] = this.ReadByte();
        }
        return bytes;
    }

    public override ushort ReadUInt16()
    {
        byte[] bytes = ReadBytes(2);
        return BitConverter.ToUInt16(bytes, 0);
    }

    public override uint ReadUInt32()
    {
        byte[] bytes = ReadBytes(4);
        return BitConverter.ToUInt32(bytes, 0);
    }

    public override ulong ReadUInt64()
    {
        byte[] bytes = ReadBytes(8);
        return BitConverter.ToUInt64(bytes, 0);
    }
}

Answer 2

I don't believe there's anything in the framework for this, no. Basically you'd need to write a class to wrap a BinaryWriter (or just a stream) and "the byte written so far" and the number of bits written. When the number of bits gets to 8, write the byte to the underlying stream and clear.

EDIT: the OP posted a possible and working implementation of the above suggestion below.

Answer 3

If you keep your data in a byte array (bools are of no use and take too much space, if you use them for bits, they take up a byte in memory) or in an array of a particular struct that fits your dataformat.

Once you have an internal memory representation, you don't need a bit-based binary writer anymore. You can simply write the data to a BinaryWriter and you're done with it.

...but the default .Net 4.0 BinaryWriter writes a Boolean value as a full byte, and as you can imagine, that negates the use of a bit-based format....

The reason for this is: the bool is, by definition, of 1 byte size in C#. The BinaryWriter simply writes what you give it.

Answer 4

I found myself in need of this as well, so I built upon OP and filled in all the read/writes (except char & string since those are a bit special).

I also made a quick unit test try it out. For streams containing only boolean (or other custom sub-byte value types) it's obviously 87.5% cheaper, and for a random mixed stream containing 75% boolean values, it was about 33% cheaper. So could be useful for some scenarios.

Here are the both classes in case anyone else needs them, use at your own risk:

/// <summary>
/// A binary writer that packs data into bits, to preserve space when using many bit/boolean values. Up to about 87.5% cheaper for streams that only contains boolean values.
/// By: jsmars@gmail.com, based on posters classes in this post: https://stackoverflow.com/questions/7051939/bit-based-binarywriter-in-c-sharp
/// </summary>
public class BinaryBitWriter : BinaryWriter
{
    public byte BitPosition { get; private set; } = 0;
    private bool[] curByte = new bool[8];
    private System.Collections.BitArray ba;

    public BinaryBitWriter(Stream s) : base(s) { }

    public override void Flush()
    {
        flushBitBuffer();
        base.Flush();
    }

    public override void Write(byte[] buffer, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            Write((byte)buffer[i]);
    }
    public override void Write(byte value)
    {
        ba = new BitArray(new byte[] { value });
        for (byte i = 0; i < 8; i++)
            Write(ba[i]);
    }
    public override void Write(bool value)
    {
        curByte[BitPosition] = value;
        BitPosition++;

        if (BitPosition == 8)
            flushBitBuffer();
    }
    public override void Write(char[] chars, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            Write(chars[i]);
    }
    public override void Write(string value)
    {
        // write strings as normal for now, so flush the bits first
        flushBitBuffer();
        base.Write(value);
    }
    public override void Write(decimal value)
    {
        var ints = decimal.GetBits(value);
        for (int i = 0; i < ints.Length; i++)
            Write(ints[i]);
    }
    public override void Write(float value) => Write(BitConverter.GetBytes(value));
    public override void Write(ulong value) => Write(BitConverter.GetBytes(value));
    public override void Write(long value) => Write(BitConverter.GetBytes(value));
    public override void Write(uint value) => Write(BitConverter.GetBytes(value));
    public override void Write(int value) => Write(BitConverter.GetBytes(value));
    public override void Write(ushort value) => Write(BitConverter.GetBytes(value));
    public override void Write(short value) => Write(BitConverter.GetBytes(value));
    public override void Write(double value) => Write(BitConverter.GetBytes(value));
    public override void Write(char[] value) => Write(value, 0, value.Length);
    public override void Write(char value)
    {
        // write strings as normal for now, so flush the bits first
        flushBitBuffer();
        base.Write(value);
        //var b = BitConverter.GetBytes(value);
        //Write(b);
    }
    public override void Write(byte[] buffer) => Write(buffer, 0, buffer.Length);
    public override void Write(sbyte value) => Write((byte)value);

    void flushBitBuffer()
    {
        if (BitPosition == 0) // Nothing to flush
            return;

        base.Write(ConvertToByte(curByte));
        BitPosition = 0;
        curByte = new bool[8];
    }

    private static byte ConvertToByte(bool[] bools)
    {
        byte b = 0;

        byte bitIndex = 0;
        for (int i = 0; i < 8; i++)
        {
            if (bools[i])
                b |= (byte)(((byte)1) << bitIndex);
            bitIndex++;
        }

        return b;
    }
}

public class BinaryBitReader : BinaryReader
{
    public byte BitPosition { get; private set; } = 8;
    private bool[] curByte = new bool[8];

    public BinaryBitReader(Stream s) : base(s)
    {

    }

    public override bool ReadBoolean()
    {
        if (BitPosition == 8)
        {
            var ba = new BitArray(new byte[] { base.ReadByte() });
            ba.CopyTo(curByte, 0);
            BitPosition = 0;
        }

        bool b = curByte[BitPosition];
        BitPosition++;
        return b;
    }

    public override byte ReadByte()
    {
        bool[] bar = new bool[8];
        byte i;
        for (i = 0; i < 8; i++)
        {
            bar[i] = this.ReadBoolean();
        }

        byte b = 0;
        byte bitIndex = 0;
        for (i = 0; i < 8; i++)
        {
            if (bar[i])
            {
                b |= (byte)(((byte)1) << bitIndex);
            }
            bitIndex++;
        }
        return b;
    }

    public override byte[] ReadBytes(int count)
    {
        byte[] bytes = new byte[count];
        for (int i = 0; i < count; i++)
        {
            bytes[i] = this.ReadByte();
        }
        return bytes;
    }


    //public override int Read() => BitConverter.ToUInt64(ReadBytes(8), 0);
    public override int Read(byte[] buffer, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            buffer[i] = ReadByte();
        return count; // we can return this here, it will die at the above row if anything is off
    }
    public override int Read(char[] buffer, int index, int count)
    {
        for (int i = index; i < index + count; i++)
            buffer[i] = ReadChar();
        return count; // we can return this here, it will die at the above row if anything is off
    }
    public override char ReadChar()
    {
        BitPosition = 8;
        return base.ReadChar();
        //BitConverter.ToChar(ReadBytes(2), 0);
    }
    public override char[] ReadChars(int count)
    {
        var chars = new char[count];
        Read(chars, 0, count);
        return chars;
    }
    public override decimal ReadDecimal()
    {
        int[] ints = new int[4];
        for (int i = 0; i < ints.Length; i++)
            ints[i] = ReadInt32();
        return new decimal(ints);
    }
    public override double ReadDouble() => BitConverter.ToDouble(ReadBytes(8), 0);
    public override short ReadInt16() => BitConverter.ToInt16(ReadBytes(2), 0);
    public override int ReadInt32() => BitConverter.ToInt32(ReadBytes(4), 0);
    public override long ReadInt64() => BitConverter.ToInt64(ReadBytes(8), 0);
    public override sbyte ReadSByte() => (sbyte)ReadByte();
    public override float ReadSingle() => BitConverter.ToSingle(ReadBytes(4), 0);
    public override string ReadString()
    {
        BitPosition = 8; // Make sure we read a new byte when we start reading the string
        return base.ReadString();
    }
    public override ushort ReadUInt16() => BitConverter.ToUInt16(ReadBytes(2), 0);
    public override uint ReadUInt32() => BitConverter.ToUInt32(ReadBytes(4), 0);
    public override ulong ReadUInt64() => BitConverter.ToUInt64(ReadBytes(8), 0);
}

And the unit tests:

public static bool UnitTest()
{
    const int testPairs = 512;

    var bitstream = new MemoryStream();
    var bitwriter = new BinaryBitWriter(bitstream);
    var bitreader = new BinaryBitReader(bitstream);

    byte[] bytes = new byte[] { 1, 2, 3, 4, 255 };
    byte Byte = 128;
    bool Bool = true;
    char[] chars = new char[] { 'a', 'b', 'c' };
    string str = "hello";
    var Float = 2.5f;
    ulong Ulong = 12345678901234567890;
    long Long = 1122334455667788;
    uint Uint = 1234567890;
    int Int = 999998888;
    ushort UShort = 12345;
    short Short = 4321;
    double Double = 9.9;
    char Char = 'A';
    sbyte Sbyte = -128;
    decimal Decimal = 10000.00001m;

    List<BBTest> pairs = new List<BBTest>();

    // Make pairs of write and read tests
    pairs.Add(new BBTest(Bool, (w) => w.Write(Bool), (r) => { if (r.ReadBoolean() != Bool) throw new Exception(); }));
    pairs.Add(new BBTest(bytes, (w) => w.Write(bytes, 0, 5), (r) => { if (arrayCompare(r.ReadBytes(5), bytes)) throw new Exception(); }));
    pairs.Add(new BBTest(Byte, (w) => w.Write(Byte), (r) => { if (r.ReadByte() != Byte) throw new Exception(); }));
    pairs.Add(new BBTest(chars, (w) => w.Write(chars, 0, 3), (r) => { if (arrayCompare(r.ReadChars(3), chars)) throw new Exception(); })); /////////////
    pairs.Add(new BBTest(str, (w) => w.Write(str), (r) => { string s; if ((s = r.ReadString()) != str) throw new Exception(); }));
    pairs.Add(new BBTest(Decimal, (w) => w.Write(Decimal), (r) => { if (r.ReadDecimal() != Decimal) throw new Exception(); }));
    pairs.Add(new BBTest(Float, (w) => w.Write(Float), (r) => { if (r.ReadSingle() != Float) throw new Exception(); }));
    pairs.Add(new BBTest(Ulong, (w) => w.Write(Ulong), (r) => { if (r.ReadUInt64() != Ulong) throw new Exception(); }));
    pairs.Add(new BBTest(Long, (w) => w.Write(Long), (r) => { if (r.ReadInt64() != Long) throw new Exception(); }));
    pairs.Add(new BBTest(Uint, (w) => w.Write(Uint), (r) => { if (r.ReadUInt32() != Uint) throw new Exception(); }));
    pairs.Add(new BBTest(Int, (w) => w.Write(Int), (r) => { if (r.ReadInt32() != Int) throw new Exception(); }));
    pairs.Add(new BBTest(UShort, (w) => w.Write(UShort), (r) => { if (r.ReadUInt16() != UShort) throw new Exception(); }));
    pairs.Add(new BBTest(Short, (w) => w.Write(Short), (r) => { if (r.ReadInt16() != Short) throw new Exception(); }));
    pairs.Add(new BBTest(Double, (w) => w.Write(Double), (r) => { if (r.ReadDouble() != Double) throw new Exception(); }));
    pairs.Add(new BBTest(Char, (w) => w.Write(Char), (r) => { if (r.ReadChar() != Char) throw new Exception(); })); ///////////////
    pairs.Add(new BBTest(bytes, (w) => w.Write(bytes), (r) => { if (arrayCompare(r.ReadBytes(5), bytes)) throw new Exception(); }));
    pairs.Add(new BBTest(Sbyte, (w) => w.Write(Sbyte), (r) => { if (r.ReadSByte() != Sbyte) throw new Exception(); }));

    // Now add all tests, and then a bunch of randomized tests, to make sure we test lots of combinations incase there is some offsetting error
    List<BBTest> test = new List<BBTest>();
    test.AddRange(pairs);
    var rnd = new Random();

    for (int i = 0; i < testPairs - test.Count; i++)
    {
        if (rnd.NextDouble() < 0.75)
            test.Add(pairs[0]);
        else
            test.Add(pairs[rnd.Next(pairs.Count)]);
    }

    // now write all the tests
    for (int i = 0; i < test.Count; i++)
        test[i].Writer(bitwriter);
    bitwriter.Flush();

    // now reset the stream and test to see that they are the same
    bitstream.Position = 0;
    for (int i = 0; i < test.Count; i++)
        test[i].ReadTest(bitreader);

    // As comparison, lets write the same stuff to a normal binarywriter and compare sized
    var binstream = new MemoryStream();
    var binwriter = new BinaryWriter(binstream);
    for (int i = 0; i < test.Count; i++)
        test[i].Writer(binwriter);
    binwriter.Flush();

    var saved = 1 - bitstream.Length / (float)binstream.Length;
    var result = $"BinaryBitWriter was {(saved * 100).ToString("0.00")}% cheaper than a normal BinaryWriter with random data";

    bool arrayCompare(IEnumerable a, IEnumerable b)
    {
        var B = b.GetEnumerator();
        B.MoveNext();
        foreach (var item in a)
        {
            if (item != B.Current)
                return false;
            B.MoveNext();
        }
        return true;
    }

    return true;
}
delegate void writer(BinaryWriter w);
delegate void reader(BinaryReader r);
class BBTest
{
    public object Object;
    public writer Writer;
    public reader ReadTest;
    public BBTest(object obj, writer w, reader r) { Object = obj; Writer = w; ReadTest = r; }
    public override string ToString() => Object.ToString();
}