gltfpack: Implement support for custom color bit count

8 bit colors are not always enough, especially since glTF uses linear
storage for vertex colors which is suboptimal - in general ~10 bits are
required for linear storage to eliminate visible banding for 8-bit sRGB
output, and more bits are required if the output is higher than 8 bits.

This change implements support for custom bit counts for color
components. To maximize the compression ratio, when bit count isn't 8 or
16, the bottom bits of the resulting value are set to 0 or 1 based on
the high bit. Varying the bit is important as this preserves 0 and 1
exactly.

A more correct scheme would involve replicating multiple top bits into
the bottom bits, but this results in significantly worse compression
ratio for N > 8, as the compressor can't exploit the corellation between
the high and low bytes. With 0 or 1 simply replicated, the byte
structure is easier to discern and the result compresses better.

gltf/gltfpack.cpp

@@ -937,15 +937,14 @@ int gltfpack(const char* input, const char* output, const char* report, Settings
 	return 0;
 }
 
-int main(int argc, char** argv)
+Settings defaults()
 {
-	meshopt_encodeIndexVersion(1);
-
 	Settings settings = {};
 	settings.quantize = true;
 	settings.pos_bits = 14;
 	settings.tex_bits = 12;
 	settings.nrm_bits = 8;
+	settings.col_bits = 8;
 	settings.trn_bits = 16;
 	settings.rot_bits = 12;
 	settings.scl_bits = 16;
@@ -954,6 +953,15 @@ int main(int argc, char** argv)
 	settings.texture_quality = 8;
 	settings.texture_scale = 1.f;
 
+	return settings;
+}
+
+int main(int argc, char** argv)
+{
+	meshopt_encodeIndexVersion(1);
+
+	Settings settings = defaults();
+
 	const char* input = 0;
 	const char* output = 0;
 	const char* report = 0;
@@ -978,6 +986,10 @@ int main(int argc, char** argv)
 		{
 			settings.nrm_bits = atoi(argv[++i]);
 		}
+		else if (strcmp(arg, "-vc") == 0 && i + 1 < argc && isdigit(argv[i + 1][0]))
+		{
+			settings.col_bits = atoi(argv[++i]);
+		}
 		else if (strcmp(arg, "-at") == 0 && i + 1 < argc && isdigit(argv[i + 1][0]))
 		{
 			settings.trn_bits = atoi(argv[++i]);
@@ -1167,6 +1179,7 @@ int main(int argc, char** argv)
 			fprintf(stderr, "\t-vp N: use N-bit quantization for positions (default: 14; N should be between 1 and 16)\n");
 			fprintf(stderr, "\t-vt N: use N-bit quantization for texture coordinates (default: 12; N should be between 1 and 16)\n");
 			fprintf(stderr, "\t-vn N: use N-bit quantization for normals and tangents (default: 8; N should be between 1 and 16)\n");
+			fprintf(stderr, "\t-vc N: use N-bit quantization for colors (default: 8; N should be between 1 and 16)\n");
 			fprintf(stderr, "\nAnimations:\n");
 			fprintf(stderr, "\t-at N: use N-bit quantization for translations (default: 16; N should be between 1 and 24)\n");
 			fprintf(stderr, "\t-ar N: use N-bit quantization for rotations (default: 12; N should be between 4 and 16)\n");

gltf/gltfpack.h

@@ -86,6 +86,7 @@ struct Settings
 	int pos_bits;
 	int tex_bits;
 	int nrm_bits;
+	int col_bits;
 
 	int trn_bits;
 	int rot_bits;

gltf/stream.cpp

@@ -221,6 +221,16 @@ static StreamFormat writeVertexStreamRaw(std::string& bin, const Stream& stream,
 	return format;
 }
 
+static int quantizeColor(float v, int bytebits, int bits)
+{
+	int result = meshopt_quantizeUnorm(v, bytebits);
+
+	// replicate the top bit into the low significant bits
+	const int mask = (1 << (bytebits - bits)) - 1;
+
+	return (result & ~mask) | (mask & -(result >> (bytebits - 1)));
+}
+
 StreamFormat writeVertexStream(std::string& bin, const Stream& stream, const QuantizationPosition& qp, const QuantizationTexture& qt, const Settings& settings)
 {
 	if (stream.type == cgltf_attribute_type_position)
@@ -445,20 +455,42 @@ StreamFormat writeVertexStream(std::string& bin, const Stream& stream, const Qua
 	}
 	else if (stream.type == cgltf_attribute_type_color)
 	{
+		int bits = settings.col_bits;
+
 		for (size_t i = 0; i < stream.data.size(); ++i)
 		{
 			const Attr& a = stream.data[i];
 
-			uint8_t v[4] = {
-			    uint8_t(meshopt_quantizeUnorm(a.f[0], 8)),
-			    uint8_t(meshopt_quantizeUnorm(a.f[1], 8)),
-			    uint8_t(meshopt_quantizeUnorm(a.f[2], 8)),
-			    uint8_t(meshopt_quantizeUnorm(a.f[3], 8))};
-			bin.append(reinterpret_cast<const char*>(v), sizeof(v));
+			if (bits > 8)
+			{
+				uint16_t v[4] = {
+				    uint16_t(quantizeColor(a.f[0], 16, bits)),
+				    uint16_t(quantizeColor(a.f[1], 16, bits)),
+				    uint16_t(quantizeColor(a.f[2], 16, bits)),
+				    uint16_t(quantizeColor(a.f[3], 16, bits))};
+				bin.append(reinterpret_cast<const char*>(v), sizeof(v));
+			}
+			else
+			{
+				uint8_t v[4] = {
+				    uint8_t(quantizeColor(a.f[0], 8, bits)),
+				    uint8_t(quantizeColor(a.f[1], 8, bits)),
+				    uint8_t(quantizeColor(a.f[2], 8, bits)),
+				    uint8_t(quantizeColor(a.f[3], 8, bits))};
+				bin.append(reinterpret_cast<const char*>(v), sizeof(v));
+			}
 		}
 
-		StreamFormat format = {cgltf_type_vec4, cgltf_component_type_r_8u, true, 4};
-		return format;
+		if (bits > 8)
+		{
+			StreamFormat format = {cgltf_type_vec4, cgltf_component_type_r_16u, true, 8};
+			return format;
+		}
+		else
+		{
+			StreamFormat format = {cgltf_type_vec4, cgltf_component_type_r_8u, true, 4};
+			return format;
+		}
 	}
 	else if (stream.type == cgltf_attribute_type_weights)
 	{