More attempts at optimizing, and fixed bug introduced in previous commit

Author: cameron314

.gitignore

@@ -26,4 +26,4 @@ benchmarks/msvc10/obj/
 benchmarks/msvc12/Debug/
 benchmarks/msvc12/Release/
 benchmarks/msvc12/obj/
-
+test/

benchmarks/bench.cpp

@@ -1,4 +1,4 @@
-// ©2013-2014 Cameron Desrochers.
+// ©2013-2015 Cameron Desrochers.
 // Distributed under the simplified BSD license (see the LICENSE file that
 // should have come with this file).
 
@@ -229,7 +229,7 @@ double runBenchmark(BenchmarkType benchmark, unsigned int randomSeed, double& ou
 		forceNoOptimizeDummy = total;
 	} break;
 	case bench_empty_remove: {
-		const counter_t MAX = 400 * 1000;
+		const counter_t MAX = 2000 * 1000;
 		out_Ops = MAX;
 		TQueue q(MAX);
 		int total = 0;
@@ -257,7 +257,7 @@ double runBenchmark(BenchmarkType benchmark, unsigned int randomSeed, double& ou
 		forceNoOptimizeDummy = total;
 	} break;
 	case bench_single_threaded: {
-		const counter_t MAX = 80 * 1000;
+		const counter_t MAX = 200 * 1000;
 		out_Ops = MAX;
 		RNG_t rng(randomSeed);
 		std::uniform_int_distribution<int> rand(0, 1);
@@ -278,16 +278,16 @@ double runBenchmark(BenchmarkType benchmark, unsigned int randomSeed, double& ou
 		forceNoOptimizeDummy = (int)(q.try_dequeue(element));
 	} break;
 	case bench_mostly_add: {
-		const counter_t MAX = 120 * 1000;
+		const counter_t MAX = 1200 * 1000;
 		out_Ops = MAX;
 		int readOps = 0;
 		RNG_t rng(randomSeed);
-		std::uniform_int_distribution<int> rand(0, 7);
+		std::uniform_int_distribution<int> rand(0, 3);
 		TQueue q(MAX);
 		int element = -1;
 		start = getSystemTime();
 		SimpleThread consumer([&]() {
-			for (counter_t i = 0; i != MAX; ++i) {
+			for (counter_t i = 0; i != MAX / 10; ++i) {
 				if (rand(rng) == 0) {
 					q.try_dequeue(element);
 					++readOps;
@@ -308,11 +308,11 @@ double runBenchmark(BenchmarkType benchmark, unsigned int randomSeed, double& ou
 		out_Ops += readOps;
 	} break;
 	case bench_mostly_remove: {
-		const counter_t MAX = 120 * 1000;
+		const counter_t MAX = 1200 * 1000;
 		out_Ops = MAX;
 		int writeOps = 0;
 		RNG_t rng(randomSeed);
-		std::uniform_int_distribution<int> rand(0, 7);
+		std::uniform_int_distribution<int> rand(0, 3);
 		TQueue q(MAX);
 		int element = -1;
 		start = getSystemTime();
@@ -323,7 +323,7 @@ double runBenchmark(BenchmarkType benchmark, unsigned int randomSeed, double& ou
 		});
 		SimpleThread producer([&]() {
 			int num = 0;
-			for (counter_t i = 0; i != MAX; ++i) {
+			for (counter_t i = 0; i != MAX / 10; ++i) {
 				if (rand(rng) == 0) {
 					q.enqueue(num);
 					++num;
@@ -338,7 +338,7 @@ double runBenchmark(BenchmarkType benchmark, unsigned int randomSeed, double& ou
 		out_Ops += writeOps;
 	} break;
 	case bench_heavy_concurrent: {
-		const counter_t MAX = 300 * 1000;
+		const counter_t MAX = 1000 * 1000;
 		out_Ops = MAX * 2;
 		TQueue q(MAX);
 		int element = -1;

benchmarks/makefile

@@ -11,7 +11,7 @@ endif
 default: benchmarks$(EXT)
 
 benchmarks$(EXT): bench.cpp ../readerwriterqueue.h ../atomicops.h ext/1024cores/spscqueue.h ext/folly/ProducerConsumerQueue.h ../tests/common/simplethread.h ../tests/common/simplethread.cpp systemtime.h systemtime.cpp makefile
-	g++ -std=c++11 -Wpedantic -Wall -DNDEBUG -O3 $(PLATFORM_OPTS) bench.cpp ../tests/common/simplethread.cpp systemtime.cpp -o benchmarks$(EXT) -pthread
+	g++ -std=c++11 -Wpedantic -Wall -DNDEBUG -O3 -g $(PLATFORM_OPTS) bench.cpp ../tests/common/simplethread.cpp systemtime.cpp -o benchmarks$(EXT) -pthread
 
 run: benchmarks$(EXT)
 	./benchmarks$(EXT)

readerwriterqueue.h

@@ -1,4 +1,4 @@
-// ©2013 Cameron Desrochers.
+// ©2013-2015 Cameron Desrochers.
 // Distributed under the simplified BSD license (see the license file that
 // should have come with this header).
 
@@ -75,6 +75,8 @@ class ReaderWriterQueue
 		assert(MAX_BLOCK_SIZE == ceilToPow2(MAX_BLOCK_SIZE) && "MAX_BLOCK_SIZE must be a power of 2");
 		assert(MAX_BLOCK_SIZE >= 2 && "MAX_BLOCK_SIZE must be at least 2");
 
+		Block* firstBlock = nullptr;
+		
 		largestBlockSize = ceilToPow2(maxSize + 1);		// We need a spare slot to fit maxSize elements in the block
 		if (largestBlockSize > MAX_BLOCK_SIZE * 2) {
 			// We need a spare block in case the producer is writing to a different block the consumer is reading from, and
@@ -84,7 +86,6 @@ class ReaderWriterQueue
 			// number of blocks - 1. Solving for maxSize and applying a ceiling to the division gives us (after simplifying):
 			size_t initialBlockCount = (maxSize + MAX_BLOCK_SIZE * 2 - 3) / (MAX_BLOCK_SIZE - 1);
 			largestBlockSize = MAX_BLOCK_SIZE;
-			Block* firstBlock = nullptr;
 			Block* lastBlock = nullptr;
 			for (size_t i = 0; i != initialBlockCount; ++i) {
 				auto block = make_block(largestBlockSize);
@@ -100,18 +101,16 @@ class ReaderWriterQueue
 				lastBlock = block;
 				block->next = firstBlock;
 			}
-			frontBlock = firstBlock;
-			tailBlock = lastBlock;
 		}
 		else {
-			auto firstBlock = make_block(largestBlockSize);
+			firstBlock = make_block(largestBlockSize);
 			if (firstBlock == nullptr) {
 				throw std::bad_alloc();
 			}
 			firstBlock->next = firstBlock;
-			frontBlock = firstBlock;
-			tailBlock = firstBlock;
 		}
+		frontBlock = firstBlock;
+		tailBlock = firstBlock;
 
 		// Make sure the reader/writer threads will have the initialized memory setup above:
 		fence(memory_order_sync);
@@ -132,7 +131,7 @@ class ReaderWriterQueue
 			size_t blockFront = block->front;
 			size_t blockTail = block->tail;
 
-			for (size_t i = blockFront; i != blockTail; i = (i + 1) & block->sizeMask()) {
+			for (size_t i = blockFront; i != blockTail; i = (i + 1) & block->sizeMask) {
 				auto element = reinterpret_cast<T*>(block->data + i * sizeof(T));
 				element->~T();
 				(void)element;
@@ -208,11 +207,10 @@ class ReaderWriterQueue
 		// block has advanced.
 
 		Block* frontBlock_ = frontBlock.load();
-		Block* tailBlock_;
 		size_t blockTail = frontBlock_->localTail;
 		size_t blockFront = frontBlock_->front.load();
 
-		if (blockFront != blockTail || blockFront != (blockTail = frontBlock_->localTail = frontBlock_->tail.load())) {
+		if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) {
 			fence(memory_order_acquire);
 
 		non_empty_front_block:
@@ -221,12 +219,12 @@ class ReaderWriterQueue
 			result = std::move(*element);
 			element->~T();
 
-			blockFront = (blockFront + 1) & frontBlock_->sizeMask();
+			blockFront = (blockFront + 1) & frontBlock_->sizeMask;
 
 			fence(memory_order_release);
 			frontBlock_->front = blockFront;
 		}
-		else if (frontBlock_ != (tailBlock_ = tailBlock.load())) {
+		else if (frontBlock_ != tailBlock.load()) {
 			fence(memory_order_acquire);
 
 			frontBlock_ = frontBlock.load();
@@ -265,7 +263,7 @@ class ReaderWriterQueue
 			result = std::move(*element);
 			element->~T();
 
-			nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask();
+			nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask;
 
 			fence(memory_order_release);
 			frontBlock_->front = nextBlockFront;
@@ -292,16 +290,15 @@ class ReaderWriterQueue
 		// See try_dequeue() for reasoning
 
 		Block* frontBlock_ = frontBlock.load();
-		Block* tailBlock_;
 		size_t blockTail = frontBlock_->localTail;
 		size_t blockFront = frontBlock_->front.load();
 
-		if (blockFront != blockTail || blockFront != (blockTail = frontBlock_->localTail = frontBlock_->tail.load())) {
+		if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) {
 			fence(memory_order_acquire);
 		non_empty_front_block:
 			return reinterpret_cast<T*>(frontBlock_->data + blockFront * sizeof(T));
 		}
-		else if (frontBlock_ != (tailBlock_ = tailBlock.load())) {
+		else if (frontBlock_ != tailBlock.load()) {
 			fence(memory_order_acquire);
 			frontBlock_ = frontBlock.load();
 			blockTail = frontBlock_->localTail = frontBlock_->tail.load();
@@ -335,23 +332,22 @@ class ReaderWriterQueue
 		// See try_dequeue() for reasoning
 
 		Block* frontBlock_ = frontBlock.load();
-		Block* tailBlock_;
 		size_t blockTail = frontBlock_->localTail;
 		size_t blockFront = frontBlock_->front.load();
 
-		if (blockFront != blockTail || blockFront != (blockTail = frontBlock_->localTail = frontBlock_->tail.load())) {
+		if (blockFront != blockTail || blockFront != (frontBlock_->localTail = frontBlock_->tail.load())) {
 			fence(memory_order_acquire);
 
 		non_empty_front_block:
 			auto element = reinterpret_cast<T*>(frontBlock_->data + blockFront * sizeof(T));
 			element->~T();
 
-			blockFront = (blockFront + 1) & frontBlock_->sizeMask();
+			blockFront = (blockFront + 1) & frontBlock_->sizeMask;
 
 			fence(memory_order_release);
 			frontBlock_->front = blockFront;
 		}
-		else if (frontBlock_ != (tailBlock_ = tailBlock.load())) {
+		else if (frontBlock_ != tailBlock.load()) {
 			fence(memory_order_acquire);
 			frontBlock_ = frontBlock.load();
 			blockTail = frontBlock_->localTail = frontBlock_->tail.load();
@@ -380,7 +376,7 @@ class ReaderWriterQueue
 			auto element = reinterpret_cast<T*>(frontBlock_->data + nextBlockFront * sizeof(T));
 			element->~T();
 
-			nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask();
+			nextBlockFront = (nextBlockFront + 1) & frontBlock_->sizeMask;
 
 			fence(memory_order_release);
 			frontBlock_->front = nextBlockFront;
@@ -404,7 +400,7 @@ class ReaderWriterQueue
 			fence(memory_order_acquire);
 			size_t blockFront = block->front.load();
 			size_t blockTail = block->tail.load();
-			result += (blockTail - blockFront) & block->sizeMask();
+			result += (blockTail - blockFront) & block->sizeMask;
 			block = block->next.load();
 		} while (block != frontBlock_);
 		return result;
@@ -432,8 +428,8 @@ class ReaderWriterQueue
 		size_t blockFront = tailBlock_->localFront;
 		size_t blockTail = tailBlock_->tail.load();
 
-		size_t nextBlockTail = (blockTail + 1) & tailBlock_->sizeMask();
-		if (nextBlockTail != blockFront || nextBlockTail != (blockFront = tailBlock_->localFront = tailBlock_->front.load())) {
+		size_t nextBlockTail = (blockTail + 1) & tailBlock_->sizeMask;
+		if (nextBlockTail != blockFront || nextBlockTail != (tailBlock_->localFront = tailBlock_->front.load())) {
 			fence(memory_order_acquire);
 			// This block has room for at least one more element
 			char* location = tailBlock_->data + blockTail * sizeof(T);
@@ -466,7 +462,7 @@ class ReaderWriterQueue
 				char* location = tailBlockNext->data + nextBlockTail * sizeof(T);
 				new (location) T(std::forward<U>(element));
 
-				tailBlockNext->tail = (nextBlockTail + 1) & tailBlockNext->sizeMask();
+				tailBlockNext->tail = (nextBlockTail + 1) & tailBlockNext->sizeMask;
 
 				fence(memory_order_release);
 				tailBlock = tailBlockNext;
@@ -580,14 +576,12 @@ class ReaderWriterQueue
 
 		char* data;		// Contents (on heap) are aligned to T's alignment
 
-		const size_t size;
-
-		AE_FORCEINLINE size_t sizeMask() const { return size - 1; }
+		const size_t sizeMask;
 
 
 		// size must be a power of two (and greater than 0)
 		Block(size_t const& _size, char* rawThis, char* _data)
-			: front(0), localTail(0), tail(0), localFront(0), next(nullptr), data(_data), size(_size), rawThis(rawThis)
+			: front(0), localTail(0), tail(0), localFront(0), next(nullptr), data(_data), sizeMask(_size - 1), rawThis(rawThis)
 		{
 		}
 

tests/unittests/unittests.cpp

@@ -1,4 +1,4 @@
-// ©2013 Cameron Desrochers
+// ©2013-2015 Cameron Desrochers
 // Unit tests for moodycamel::ReaderWriterQueue
 
 #include <cstdio>
@@ -53,6 +53,7 @@ class ReaderWriterQueueTests : public TestClass<ReaderWriterQueueTests>
 		REGISTER_TEST(enqueue_many);
 		REGISTER_TEST(nonempty_destroy);
 		REGISTER_TEST(try_enqueue);
+		REGISTER_TEST(try_dequeue);
 		REGISTER_TEST(peek);
 		REGISTER_TEST(pop);
 		REGISTER_TEST(size_approx);
@@ -241,6 +242,23 @@ class ReaderWriterQueueTests : public TestClass<ReaderWriterQueueTests>
 		return true;
 	}
 
+	bool try_dequeue()
+	{
+		int item;
+		
+		{
+			ReaderWriterQueue<int> q(1);
+			ASSERT_OR_FAIL(!q.try_dequeue(item));
+		}
+		
+		{
+			ReaderWriterQueue<int, 2> q(10);
+			ASSERT_OR_FAIL(!q.try_dequeue(item));
+		}
+		
+		return true;
+	}
+	
 	bool threaded()
 	{
 		weak_atomic<int> result;