--User changes

-Always force times of each flame to increase from zero when saving a file.
 -Remove check for times when doing a sequence in EmberGenome because the original times are never used there.

--Bug fixes
 -Multi-GPU synchronization was not actually thread safe and was likely doing less iters than requested. It is now properly synchronized.

--Code changes
 -Optimize Interpolater by making it a non-static class by adding some members used for caching values during interpolation.
 -Cache values in SheepTools as well, which was already a non-static class.
 -General cleanup.

Source/Ember/EmberToXml.h

@@ -78,20 +78,7 @@ public:
 			{
 				auto prev = embers.begin();
 
-				//Check to see if there are valid times by checking if any differed.
+				//Always ensure times make sense.
-				//If so, assume they were intentionally entered times.
-				for (auto it = Advance(embers.begin(), 1); it != embers.end(); ++it)
-				{
-					if (it->m_Time != prev->m_Time)
-					{
-						hasTimes = true;
-						break;
-					}
-
-					prev = it;
-				}
-
-				if (!hasTimes)
 				for (auto& ember : embers)
 					ember.m_Time = t++;
 

Source/Ember/Interpolate.h

@@ -376,7 +376,7 @@ public:
 	/// <param name="time">The time position in the vector specifying the point of interpolation</param>
 	/// <param name="stagger">Stagger if > 0</param>
 	/// <param name="result">The interpolated result</param>
-	static void Interpolate(const vector<Ember<T>>& embers, T time, T stagger, Ember<T>& result)
+	void Interpolate(const vector<Ember<T>>& embers, T time, T stagger, Ember<T>& result)
 	{
 		Interpolate(embers.data(), embers.size(), time, stagger, result);
 	}
@@ -389,7 +389,7 @@ public:
 	/// <param name="time">The time position in the vector specifying the point of interpolation</param>
 	/// <param name="stagger">Stagger if > 0</param>
 	/// <param name="result">The interpolated result</param>
-	static void Interpolate(const Ember<T>* embers, size_t size, T time, T stagger, Ember<T>& result)
+	void Interpolate(const Ember<T>* embers, size_t size, T time, T stagger, Ember<T>& result)
 	{
 		if (size == 1)
 		{
@@ -398,8 +398,6 @@ public:
 		}
 
 		size_t i1, i2;
-		vector<T> c(2);
-		Ember<T> localEmbers[4];
 		bool smoothFlag = false;
 
 		if (embers[0].m_Time >= time)
@@ -423,31 +421,31 @@ public:
 			i2 = i1 + 1;
 		}
 
-		c[0] = (embers[i2].m_Time - time) / (embers[i2].m_Time - embers[i1].m_Time);
+		m_Coeffs[0] = (embers[i2].m_Time - time) / (embers[i2].m_Time - embers[i1].m_Time);
-		c[1] = 1 - c[0];
+		m_Coeffs[1] = 1 - m_Coeffs[0];
 
 		//To interpolate the xforms, make copies of the source embers
 		//and ensure that they both have the same number of xforms before progressing.
 		if (embers[i1].m_Interp == eInterp::EMBER_INTERP_LINEAR)
 		{
-			Align(&embers[i1], &localEmbers[0], 2);
+			Align(&embers[i1], &m_Embers[0], 2);
 			smoothFlag = false;
 		}
 		else
 		{
 			if (i1 == 0)
 			{
-				Align(&embers[i1], &localEmbers[0], 2);
+				Align(&embers[i1], &m_Embers[0], 2);
 				smoothFlag = false;
 			}
 			else if (i2 == size - 1)
 			{
-				Align(&embers[i1], &localEmbers[0], 2);
+				Align(&embers[i1], &m_Embers[0], 2);
 				smoothFlag = false;
 			}
 			else
 			{
-				Align(&embers[i1 - 1], &localEmbers[0], 4);//Should really be doing some sort of checking here to ensure the ember vectors have 4 elements.
+				Align(&embers[i1 - 1], &m_Embers[0], 4);//Should really be doing some sort of checking here to ensure the ember vectors have 4 elements.
 				smoothFlag = true;
 			}
 		}
@@ -458,9 +456,9 @@ public:
 		result.m_PaletteInterp = ePaletteInterp::INTERP_HSV;
 
 		if (!smoothFlag)
-			result.Interpolate(&localEmbers[0], 2, c, stagger);
+			result.Interpolate(&m_Embers[0], 2, m_Coeffs, stagger);
 		else
-			result.InterpolateCatmullRom(&localEmbers[0], 4, c[1]);
+			result.InterpolateCatmullRom(&m_Embers[0], 4, m_Coeffs[1]);
 	}
 
 	/// <summary>
@@ -492,9 +490,9 @@ public:
 	{
 		for (size_t i = 0; i < source->TotalVariationCount(); i++)//Iterate through the first xform's variations.
 		{
-			Variation<T>* var = source->GetVariation(i);//Grab the variation at index in in the first xform.
+			auto var = source->GetVariation(i);//Grab the variation at index in in the first xform.
-			Variation<T>* var2 = dest->GetVariationById(var->VariationId());//See if the same variation exists in the second xform.
+			auto var2 = dest->GetVariationById(var->VariationId());//See if the same variation exists in the second xform.
-			ParametricVariation<T>* parVar = dynamic_cast<ParametricVariation<T>*>(var);//Parametric cast of the first var for later.
+			auto parVar = dynamic_cast<ParametricVariation<T>*>(var);//Parametric cast of the first var for later.
 
 			if (!var2)//Only take action if the second xform did not contain this variation.
 			{
@@ -502,7 +500,7 @@ public:
 				{
 					if (parVar)
 					{
-						Variation<T>* parVarCopy = parVar->Copy();
+						auto parVarCopy = parVar->Copy();
 
 						if (clearWeights)
 							parVarCopy->m_Weight = 0;
@@ -512,7 +510,7 @@ public:
 				}
 				else//Add regardless of type.
 				{
-					Variation<T>* varCopy = var->Copy();
+					auto varCopy = var->Copy();
 
 					if (clearWeights)
 						varCopy->m_Weight = 0;
@@ -732,8 +730,7 @@ public:
 			{
 				for (size_t col = 0; col < 2; col++)
 				{
-					int sym0, sym1;
+					bool sym0, sym1, padSymFlag = false;
-					int padSymFlag;
 					d = cxang[k][col] - cxang[k - 1][col];
 
 					//Adjust to avoid the -pi/pi discontinuity.
@@ -745,7 +742,6 @@ public:
 					//If this is an asymmetric case, store the NON-symmetric angle
 					//Check them pairwise and store the reference angle in the second
 					//to avoid overwriting if asymmetric on both sides.
-					padSymFlag = 0;
 					sym0 = (embers[k - 1].GetXform(xfi)->m_Animate == 0 || (embers[k - 1].GetXform(xfi)->Empty() && padSymFlag));
 					sym1 = (embers[k    ].GetXform(xfi)->m_Animate == 0 || (embers[k    ].GetXform(xfi)->Empty() && padSymFlag));
 
@@ -936,5 +932,9 @@ public:
 
 		return ad > bd;
 	}
+
+private:
+	vector<T> m_Coeffs = vector<T>(2);
+	Ember<T> m_Embers[4];
 };
 }

Source/Ember/Renderer.cpp

@@ -416,7 +416,7 @@ eRenderStatus Renderer<T, bucketT>::Run(vector<byte>& finalImage, double time, s
 	//it.Tic();
 	//Interpolate.
 	if (m_EmbersP->size() > 1)
-		Interpolater<T>::Interpolate(*m_EmbersP, T(time), 0, m_Ember);
+		m_Interpolater.Interpolate(*m_EmbersP, T(time), 0, m_Ember);
 
 	//it.Toc("Interp 1");
 
@@ -454,7 +454,7 @@ eRenderStatus Renderer<T, bucketT>::Run(vector<byte>& finalImage, double time, s
 	//Additional interpolation will be done in the temporal samples loop.
 	//it.Tic();
 	if (m_EmbersP->size() > 1)
-		Interpolater<T>::Interpolate(*m_EmbersP, deTime, 0, m_Ember);
+		m_Interpolater.Interpolate(*m_EmbersP, deTime, 0, m_Ember);
 
 	//it.Toc("Interp 2");
 	ClampGteRef<T>(m_Ember.m_MinRadDE, 0);
@@ -479,7 +479,7 @@ eRenderStatus Renderer<T, bucketT>::Run(vector<byte>& finalImage, double time, s
 		//Interpolate again.
 		//it.Tic();
 		if (TemporalSamples() > 1 && m_EmbersP->size() > 1)
-			Interpolater<T>::Interpolate(*m_EmbersP, temporalTime, 0, m_Ember);//This will perform all necessary precalcs via the ember/xform/variation assignment operators.
+			m_Interpolater.Interpolate(*m_EmbersP, temporalTime, 0, m_Ember);//This will perform all necessary precalcs via the ember/xform/variation assignment operators.
 
 		//it.Toc("Interp 3");
 

Source/Ember/Renderer.h

@@ -184,6 +184,7 @@ private:
 protected:
 	vector<Ember<T>>* m_EmbersP = &m_Embers;
 	vector<Ember<T>> m_ThreadEmbers;
+	Interpolater<T> m_Interpolater;
 	CarToRas<T> m_CarToRas;
 	unique_ptr<StandardIterator<T>> m_StandardIterator = make_unique<StandardIterator<T>>();
 	unique_ptr<XaosIterator<T>> m_XaosIterator = make_unique<XaosIterator<T>>();

Source/Ember/SheepTools.h

@@ -452,14 +452,13 @@ public:
 		else if (crossMode == eCrossMode::CROSS_INTERPOLATE)
 		{
 			//Linearly interpolate somewhere between the two.
-			Ember<T> parents[2];
 			//t = 0.5;//If you ever need to test.
 			t = m_Rand.Frand01<T>();
-			parents[0] = ember0;
+			m_Parents[0] = ember0;
-			parents[1] = ember1;
+			m_Parents[1] = ember1;
-			parents[0].m_Time = T(0);
+			m_Parents[0].m_Time = T(0);
-			parents[1].m_Time = T(1);
+			m_Parents[1].m_Time = T(1);
-			Interpolater<T>::Interpolate(parents, 2, t, 0, emberOut);
+			m_Interpolater.Interpolate(m_Parents, 2, t, 0, emberOut);
 
 			for (i = 0; i < emberOut.TotalXformCount(); i++)
 				emberOut.GetTotalXform(i)->DeleteMotionElements();
@@ -990,21 +989,20 @@ public:
 	void Edge(Ember<T>* embers, Ember<T>& result, T blend, bool seqFlag)
 	{
 		size_t i, si;
-		Ember<T> spun[2], prealign[2];
 
 		//Insert motion magic here :
 		//If there are motion elements, modify the contents of
 		//the result xforms before rotate is called.
 		for (si = 0; si < 2; si++)
 		{
-			prealign[si] = embers[si];
+			m_EdgePrealign[si] = embers[si];
 
 			for (i = 0; i < embers[si].TotalXformCount(); i++)
 			{
 				auto xform = embers[si].GetTotalXform(i);
 
 				if (!xform->m_Motion.empty())
-					xform->ApplyMotion(*(prealign[si].GetTotalXform(i)), blend);//Apply motion parameters to result.xform[i] using blend parameter.
+					xform->ApplyMotion(*(m_EdgePrealign[si].GetTotalXform(i)), blend);//Apply motion parameters to result.xform[i] using blend parameter.
 			}
 		}
 
@@ -1012,20 +1010,20 @@ public:
 		//This keeps the original interpolation type intact.
 		if (seqFlag && blend == 0)
 		{
-			result = prealign[0];
+			result = m_EdgePrealign[0];
 		}
 		else
 		{
 			//Align what's going to be interpolated.
-			Interpolater<T>::Align(prealign, spun, 2);
+			Interpolater<T>::Align(m_EdgePrealign, m_EdgeSpun, 2);
-			spun[0].m_Time = 0;
+			m_EdgeSpun[0].m_Time = 0;
-			spun[1].m_Time = 1;
+			m_EdgeSpun[1].m_Time = 1;
 			//Call this first to establish the asymmetric reference angles.
-			Interpolater<T>::AsymmetricRefAngles(spun, 2);
+			Interpolater<T>::AsymmetricRefAngles(m_EdgeSpun, 2);
 			//Rotate the aligned xforms.
-			spun[0].RotateAffines(-blend * 360);
+			m_EdgeSpun[0].RotateAffines(-blend * 360);
-			spun[1].RotateAffines(-blend * 360);
+			m_EdgeSpun[1].RotateAffines(-blend * 360);
-			Interpolater<T>::Interpolate(spun, 2, m_Smooth ? Interpolater<T>::Smoother(blend) : blend, m_Stagger, result);
+			m_Interpolater.Interpolate(m_EdgeSpun, 2, m_Smooth ? Interpolater<T>::Smoother(blend) : blend, m_Stagger, result);
 		}
 
 		//Make sure there are no motion elements in the result.
@@ -1340,6 +1338,10 @@ private:
 	vector<uint> m_Hist;
 	EmberToXml<T> m_EmberToXml;
 	Iterator<T>* m_Iterator;
+	Interpolater<T> m_Interpolater;
+	Ember<T> m_Parents[2];
+	Ember<T> m_EdgeSpun[2];
+	Ember<T> m_EdgePrealign[2];
 	unique_ptr<StandardIterator<T>> m_StandardIterator = make_unique<StandardIterator<T>>();
 	unique_ptr<XaosIterator<T>> m_XaosIterator = make_unique<XaosIterator<T>>();
 	unique_ptr<Renderer<T, bucketT>> m_Renderer;

Source/Ember/Utils.h

@@ -1028,6 +1028,26 @@ static vector<string> Split(const string& str, char del)
 	return vec;
 }
 
+/// <summary>
+/// Thin wrapper around joining a thread.
+/// </summary>
+/// <param name="th">The thread to join</param>
+static void Join(std::thread& th)
+{
+	if (th.joinable())
+		th.join();
+}
+
+/// <summary>
+/// Thin wrapper around joining a vector of threads.
+/// </summary>
+/// <param name="vec">The vector of threads to join</param>
+static void Join(std::vector<std::thread>& vec)
+{
+	for (auto& it : vec)
+		Join(it);
+}
+
 /// <summary>
 /// Return a character pointer to a version string composed of the EMBER_OS and EMBER_VERSION values.
 /// </summary>

Source/EmberAnimate/EmberAnimate.cpp

@@ -35,6 +35,7 @@ bool EmberAnimate(EmberOptions& opt)
 	vector<Ember<T>> embers;
 	XmlToEmber<T> parser;
 	EmberToXml<T> emberToXml;
+	Interpolater<T> interpolater;
 	EmberReport emberReport;
 	const vector<pair<size_t, size_t>> devices = Devices(opt.Devices());
 	std::atomic<size_t> atomfTime;
@@ -275,7 +276,7 @@ bool EmberAnimate(EmberOptions& opt)
 			opt.FirstFrame(size_t(embers[0].m_Time));
 
 		if (opt.LastFrame() == UINT_MAX)
-			opt.LastFrame(ClampGte<size_t>(size_t(embers.back().m_Time),//Make sure time - 1 is positive before converting to size_t.
+			opt.LastFrame(ClampGte<size_t>(size_t(embers.back().m_Time),
 										   opt.FirstFrame() + opt.Dtime()));//Make sure the final value is at least first frame + dtime.
 	}
 
@@ -340,9 +341,20 @@ bool EmberAnimate(EmberOptions& opt)
 		std::thread writeThread;
 		os.imbue(std::locale(""));
 
-		while (atomfTime.fetch_add(opt.Dtime()), ((ftime = atomfTime.load()) <= opt.LastFrame()))
+		//The conditions of this loop use atomics to synchronize when running on multiple GPUs.
+		//The order is reversed from the usual loop: rather than compare and increment the counter,
+		//it's incremented, then compared. This is done to ensure the GPU on this thread "claims" this
+		//frame before working on it.
+		//The mechanism for incrementing is:
+		//	Do an atomic add, which returns the previous value.
+		//	Add the time increment Dtime() to the return value to mimic what the new atomic value should be.
+		//	Assign the result to the ftime counter.
+		//	Do a <= comparison to LastFrame().
+		//		If true, enter the loop and immediately decrement the counter by Dtime() to make up for the fact
+		//		that it was first incremented before comparing.
+		while ((ftime = (atomfTime.fetch_add(opt.Dtime()) + opt.Dtime())) <= opt.LastFrame())
 		{
-			T localTime = T(ftime) - 1;
+			T localTime = T(ftime) - opt.Dtime();
 
 			if (opt.Verbose() && ((opt.LastFrame() - opt.FirstFrame()) / opt.Dtime() >= 1))
 			{
@@ -374,7 +386,7 @@ bool EmberAnimate(EmberOptions& opt)
 					cout << "Writing " << flameName << "\n";
 				}
 
-				Interpolater<T>::Interpolate(embers, localTime, 0, centerEmber);//Get center flame.
+				interpolater.Interpolate(embers, localTime, 0, centerEmber);//Get center flame.
 				emberToXml.Save(flameName, centerEmber, opt.PrintEditDepth(), true, opt.HexPalette(), true, false, false);
 				centerEmber.Clear();
 			}
@@ -400,9 +412,7 @@ bool EmberAnimate(EmberOptions& opt)
 
 			//Run image writing in a thread. Although doing it this way duplicates the final output memory, it saves a lot of time
 			//when running with OpenCL. Call join() to ensure the previous thread call has completed.
-			if (writeThread.joinable())
+			Join(writeThread);
-				writeThread.join();
-
 			auto threadVecIndex = finalImageIndex;//Cache before launching thread.
 
 			if (opt.ThreadedWrite())//Copies are passed of all but the first parameter to saveFunc(), to avoid conflicting with those values changing when starting the render for the next image.
@@ -414,8 +424,7 @@ bool EmberAnimate(EmberOptions& opt)
 				saveFunc(finalImages[threadVecIndex], filename, comments, renderer->FinalRasW(), renderer->FinalRasH(), renderer->NumChannels());//Will always use the first index, thereby not requiring more memory.
 		}
 
-		if (writeThread.joinable())//One final check to make sure all writing is done before exiting this thread.
+		Join(writeThread);//One final check to make sure all writing is done before exiting this thread.
-			writeThread.join();
 	};
 	threadVec.reserve(renderers.size());
 
@@ -427,10 +436,7 @@ bool EmberAnimate(EmberOptions& opt)
 		}, r));
 	}
 
-	for (auto& th : threadVec)
+	Join(threadVec);
-		if (th.joinable())
-			th.join();
-
 	t.Toc("\nFinished in: ", true);
 	return true;
 }

Source/EmberCL/RendererCL.cpp

@@ -929,9 +929,7 @@ bool RendererCL<T, bucketT>::BuildIterProgramForEmber(bool doAccum)
 				func(m_Devices[device].get());
 		}
 
-		for (auto& th : threads)
+		Join(threads);
-			if (th.joinable())
-				th.join();
 
 		if (b)
 		{
@@ -992,7 +990,7 @@ bool RendererCL<T, bucketT>::RunIter(size_t iterCount, size_t temporalSample, si
 		auto& wrapper = m_Devices[dev]->m_Wrapper;
 		intmax_t itersRemaining = 0;
 
-		while (atomLaunchesRan.fetch_add(1), (b && (atomLaunchesRan.load() <= launches) && ((itersRemaining = atomItersRemaining.load()) > 0) && !m_Abort))
+		while (b && (atomLaunchesRan.fetch_add(1) + 1 <= launches) && ((itersRemaining = atomItersRemaining.load()) > 0) && !m_Abort)
 		{
 			cl_uint argIndex = 0;
 #ifdef TEST_CL
@@ -1102,10 +1100,7 @@ bool RendererCL<T, bucketT>::RunIter(size_t iterCount, size_t temporalSample, si
 			iterFunc(device, index);
 	}
 
-	for (auto& th : threadVec)
+	Join(threadVec);
-		if (th.joinable())
-			th.join();
-
 	itersRan = atomItersRan.load();
 
 	if (m_Devices.size() > 1)//Determine whether/when to sum histograms of secondary devices with the primary.

Source/EmberCommon/EmberOptions.h

@@ -145,11 +145,9 @@ private:
 	/// </summary>
 	EmberOptionEntry()
 	{
-		m_OptionUse = eOptionUse::OPT_USE_ALL;
-		m_Option.nArgType = SO_NONE;
 		m_Option.nId = 0;
 		m_Option.pszArg = _T("--fillmein");
-		m_DocString = "Dummy doc";
+		m_Option.nArgType = SO_NONE;
 	}
 
 public:
@@ -203,13 +201,13 @@ public:
 	/// <summary>
 	/// Functor accessors.
 	/// </summary>
-	inline T    operator() (void) { return m_Val; }
+	inline T    operator() (void) const { return m_Val; }
 	inline void operator() (T t) { m_Val = t; }
 
 private:
-	eOptionUse m_OptionUse;
+	eOptionUse m_OptionUse = eOptionUse::OPT_USE_ALL;
 	CSimpleOpt::SOption m_Option;
-	string m_DocString;
+	string m_DocString = "Dummy doc";
 	string m_NameWithoutDashes;
 	T m_Val;
 };

Source/EmberGenome/EmberGenome.cpp

@@ -117,6 +117,7 @@ bool EmberGenome(EmberOptions& opt)
 	Ember<T>* aselp0, *aselp1, *pTemplate = nullptr;
 	XmlToEmber<T> parser;
 	EmberToXml<T> emberToXml;
+	Interpolater<T> interpolater;
 	EmberReport emberReport, emberReport2;
 	const vector<pair<size_t, size_t>> devices = Devices(opt.Devices());
 	auto progress = make_unique<RenderProgress<T>>();
@@ -370,7 +371,7 @@ bool EmberGenome(EmberOptions& opt)
 
 			if (!exactTimeMatch)
 			{
-				Interpolater<T>::Interpolate(embers, T(ftime), T(opt.Stagger()), interpolated);
+				interpolater.Interpolate(embers, T(ftime), T(opt.Stagger()), interpolated);
 
 				for (i = 0; i < embers.size(); i++)
 				{
@@ -397,31 +398,20 @@ bool EmberGenome(EmberOptions& opt)
 
 	if (opt.Sequence() != "")
 	{
-		frame = std::max(opt.Frame(), opt.Time());
-
 		if (opt.Frames() == 0)
 		{
 			cerr << "nframes must be positive and non-zero, not " << opt.Frames() << ".\n";
 			return false;
 		}
 
-		for (i = 0; i < embers.size(); i++)
-		{
-			if (i > 0 && embers[i].m_Time <= embers[i - 1].m_Time)
-			{
-				cerr << "Error: control points must be sorted by time, but time " << embers[i].m_Time << " <= " << embers[i - 1].m_Time << ", index " << i << ".\n";
-				return false;
-			}
-		}
-
 		if (opt.Enclosed())
 			cout << "<sequence version=\"EMBER-" << EmberVersion() << "\">\n";
 
-		spread = 1 / T(opt.Frames());
 		frameCount = 0;
 		os.str("");
 		os << setfill('0');
 		auto padding = streamsize(std::log10(((opt.Frames() * opt.Loops()) + opt.Frames()) * embers.size())) + 1;
+		t.Tic();
 
 		for (i = 0; i < embers.size(); i++)
 		{
@@ -451,7 +441,7 @@ bool EmberGenome(EmberOptions& opt)
 
 				for (frame = 0; frame < opt.Frames(); frame++)
 				{
-					seqFlag = (frame == 0 || frame == opt.Frames() - 1);
+					seqFlag = (frame == 0 || (frame == opt.Frames() - 1));
 					blend = frame / T(opt.Frames());
 					result.Clear();
 					tools.SpinInter(&embers[i], pTemplate, result, frameCount++, seqFlag, blend);
@@ -465,6 +455,7 @@ bool EmberGenome(EmberOptions& opt)
 		tools.Spin(embers.back(), pTemplate, result, frameCount, 0);
 		FormatName(result, os, padding);
 		cout << emberToXml.ToString(result, opt.Extras(), opt.PrintEditDepth(), !opt.NoEdits(), opt.HexPalette());
+		t.Toc("Sequencing");
 
 		if (opt.Enclosed())
 			cout << "</sequence>\n";

Source/EmberTester/EmberTester.cpp

@@ -1866,8 +1866,8 @@ void TestThreadedKernel()
 					cout << "Successful run inside thread 2..." << endl;
 				}
 			});
-			th1.join();
+			Join(th1);
-			th2.join();
+			Join(th2);
 			cout << "Successful join of kernel thread..." << endl;
 		}
 	}

Source/Fractorium/FinalRenderEmberController.cpp

@@ -230,15 +230,27 @@ FinalRenderEmberController<T>::FinalRenderEmberController(FractoriumFinalRenderD
 					renderer->SetExternalEmbersPointer(&embers);//All will share a pointer to the original vector to conserve memory with large files. Ok because the vec doesn't get modified.
 
 					//Render each image, cancelling if m_Run ever gets set to false.
-					while (atomfTime.fetch_add(1), ((ftime = atomfTime.load() - 1) < m_EmberFile.Size()) && m_Run)//Needed to set 1 to claim this iter from other threads, so decrement it to be zero-indexed here.
+					//Render each image, cancelling if m_Run ever gets set to false.
+					//The conditions of this loop use atomics to synchronize when running on multiple GPUs.
+					//The order is reversed from the usual loop: rather than compare and increment the counter,
+					//it's incremented, then compared. This is done to ensure the GPU on this thread "claims" this
+					//frame before working on it.
+					//The mechanism for incrementing is:
+					//	Do an atomic add, which returns the previous value.
+					//	Add 1 to the return value to mimic what the new atomic value should be.
+					//	Assign the result to the ftime counter.
+					//	Do a <= comparison to m_EmberFile.Size() and check m_Run.
+					//		If true, enter the loop and immediately decrement the counter by 1 to make up for the fact
+					//		that it was first incremented before comparing.
+					while (((ftime = (atomfTime.fetch_add(1) + 1)) <= m_EmberFile.Size()) && m_Run)//Needed to set 1 to claim this iter from other threads, so decrement it below to be zero-indexed here.
 					{
-						T localTime = T(ftime);
+						--ftime;
 						Output("Image " + ToString(ftime + 1ULL) + ":\n" + ComposePath(QString::fromStdString(m_EmberFile.Get(ftime)->m_Name)));
 						renderer->Reset();//Have to manually set this since the ember is not set each time through.
 						renderTimer.Tic();//Toc() is called in RenderComplete().
 
 						//Can't use strips render here. Run() must be called directly for animation.
-						if (renderer->Run(finalImages[finalImageIndex], localTime) != eRenderStatus::RENDER_OK)
+						if (renderer->Run(finalImages[finalImageIndex], T(ftime)) != eRenderStatus::RENDER_OK)
 						{
 							Output("Rendering failed.\n");
 							m_Fractorium->ErrorReportToQTextEdit(renderer->ErrorReport(), m_FinalRenderDialog->ui.FinalRenderTextOutput, false);//Internally calls invoke.
@@ -247,9 +259,7 @@ FinalRenderEmberController<T>::FinalRenderEmberController(FractoriumFinalRenderD
 						}
 						else
 						{
-							if (writeThread.joinable())
+							Join(writeThread);
-								writeThread.join();
-
 							stats = renderer->Stats();
 							comments = renderer->ImageComments(stats, 0, true);
 							writeThread = std::thread([&](size_t tempTime, size_t threadFinalImageIndex)
@@ -272,8 +282,7 @@ FinalRenderEmberController<T>::FinalRenderEmberController(FractoriumFinalRenderD
 						finalImageIndex ^= 1;//Toggle the index.
 					}
 
-					if (writeThread.joinable())//One final check to make sure all writing is done before exiting this thread.
+					Join(writeThread);//One final check to make sure all writing is done before exiting this thread.
-						writeThread.join();
 				};
 				threadVec.reserve(m_Renderers.size());
 
@@ -285,10 +294,7 @@ FinalRenderEmberController<T>::FinalRenderEmberController(FractoriumFinalRenderD
 					}, r));
 				}
 
-				for (auto& th : threadVec)
+				Join(threadVec);
-					if (th.joinable())
-						th.join();
-
 				HandleFinishedProgress();//One final check that all images were finished.
 			}
 			else if (m_Renderer.get())//Make sure a renderer was created and render all images, but not as an animation sequence (without temporal samples motion blur).