Allow larger diffs when removing too points in a row

It happened today that a large delay (27 minutes!) caused the
temperature to spike too much, and for too long, that when it returned
to normal, the difference compared to before was too large. This led to
everything being excluded after the spike, and the graphs not showing
anything.

Allowing larger spikes after removing too many in a row doesn't solve it
perfectly, but it makes it better.

web-view/src/js/cleanData.test.ts

@@ -143,6 +143,74 @@ describe('Clean data', () => {
 
 			const actual = removeSpikes(input);
 			should(actual).deepEqual(input);
-		})
+		});
+
+		it('Does not remove too many spikes', () => {
+			// Test that it doesn't keep removing more and more spikes, such
+			// as if there was a power outage, or the temperature actually
+			// started to rise quickly.
+
+			// Values taken from 2024-04-26 07:49 to 10:12 EDT.
+			const input: TimePoint[] = [
+				{ time: 1, value: 0.95 },
+				{ time: 2, value: -0.52 },
+				{ time: 3, value: -0.88 },
+				{ time: 4, value: -0.82 },
+				{ time: 5, value: -0.74 },
+				{ time: 6, value: -0.5 },
+				{ time: 7, value: 3.36 },
+				{ time: 8, value: 7.98 }, // Some delay happened here.
+				{ time: 9, value: 8.3 }, // And continued to affect here.
+				{ time: 10, value: 3.94 }, // Seemingly normal temperature.
+				{ time: 11, value: 7.23 }, // Long delay happened here.
+				{ time: 12, value: 15.04 }, // But was really resolved here.
+				{ time: 13, value: 8.86 }, // Device cooled back down.
+				{ time: 14, value: 6.41 }, // Normal temperatures mostly resumed here.
+				{ time: 15, value: 5.72 },
+				{ time: 16, value: 5.77 },
+				{ time: 17, value: 5.85 },
+				{ time: 18, value: 6.07 },
+				{ time: 19, value: 6.33 },
+				{ time: 20, value: 6.64 },
+				{ time: 21, value: 6.99 },
+				{ time: 22, value: 7.26 },
+				{ time: 23, value: 7.54 },
+				{ time: 24, value: 7.84 },
+				{ time: 25, value: 10.09 },
+			];
+			const expected: TimePoint[] = [
+				{ time: 1, value: 0.95 },
+				{ time: 2, value: -0.52 },
+				{ time: 3, value: -0.88 },
+				{ time: 4, value: -0.82 },
+				{ time: 5, value: -0.74 },
+				{ time: 6, value: -0.5 },
+				{ time: 7, value: 3.36 },
+				// These two are definite spikes, but the difference to #7 is within normal range.
+				{ time: 8, value: 7.98 }, // Some delay happened here.
+				{ time: 9, value: 8.3 }, // And continued to affect here.
+				// This one should be included, but the "compare to previous diff" sees too big of a drop.
+				// TODO: Figure out how to include this one.
+				// { time: 10, value: 3.94 },
+				{ time: 11, value: 7.23 }, // Long delay happened here.
+				// Remove 12, since it was way too big.
+				{ time: 13, value: 8.86 }, // Device cooled back down.
+				{ time: 14, value: 6.41 }, // Normal temperatures mostly resumed here.
+				{ time: 15, value: 5.72 },
+				{ time: 16, value: 5.77 },
+				{ time: 17, value: 5.85 },
+				{ time: 18, value: 6.07 },
+				{ time: 19, value: 6.33 },
+				{ time: 20, value: 6.64 },
+				{ time: 21, value: 6.99 },
+				{ time: 22, value: 7.26 },
+				{ time: 23, value: 7.54 },
+				{ time: 24, value: 7.84 },
+				{ time: 25, value: 10.09 },
+			];
+
+			const actual = removeSpikes(input);
+			should(actual).deepEqual(expected);
+		});
 	});
 });

web-view/src/js/cleanData.ts

@@ -145,8 +145,14 @@ export function removeSpikes(
 		return data;
 	}
 
+	let removed: TimePoint[] = [];
+	let currMinChange = minChange;
+
 	for(let i = 0; i < data.length; i++) {
-		let prevDiff:number, thisDiff:number;
+		let prevDiff:number,
+			thisDiff:number,
+			thisValue: number,
+			prevValue: number;
 
 		// General case for most of an array
 		if(i > 2) {
@@ -158,14 +164,32 @@ export function removeSpikes(
 			thisDiff = Math.abs(data[i].value - data[i + 1].value);
 		}
 		const isCooling = i > 0 && data[i].value < data[i - 1].value;
-		const minChangeToUse = isCooling ? minChange * 2 : minChange;
+
+		// Minimum change required before removing the spike.
+		const minChangeToUse = isCooling ? currMinChange * 2 : currMinChange;
 
 		const tooBig = (thisDiff - prevDiff) > minChangeToUse
 					&& (thisDiff / prevDiff) > minDiffMultiple;
 		if(tooBig) {
-			console.log(`Removing spike element ${formatTime(data[i].time)}: Value: ${data[i].value}; prevDiff: ${prevDiff}; thisDiff: ${thisDiff}`);
+			console.log(`Removing spike element ${formatTime(data[i].time)}: Value: ${data[i].value}; Prev value: ${data[i-1].value}; prevDiff: ${prevDiff}; thisDiff: ${thisDiff}`);
+
+			// Remove the element, and store it so it can be re-added if needed.
+			removed.push(data[i]);
 			data = removeElement(data, i);
 			i--; // Removed the element, make sure we don't skip the next
+		} else {
+			// Held onto an element, reset the removed list.
+			removed = [];
+			currMinChange = minChange;
+		}
+
+		// Don't allow removing too many elements in a row.
+		if(removed.length >= 4) {
+			console.log('Removed too many elements. Going to re-add some:', removed);
+			data = insertElements(data, removed, i + 1);
+
+			currMinChange *= 1.1; // Allow reprocessed elements to be larger.
+			removed = [];
 		}
 	}
 
@@ -179,3 +203,18 @@ function removeElement<T>(arr:Array<T>, index:number):Array<T> {
 
 	return arr.slice(0, index).concat(arr.slice(index+1));
 }
+
+/**
+ * Insert a number of elements into the middle of an array.
+ * @param arr The array to add the elements to.
+ * @param toInsert The elements to add.
+ * @param index The index of `arr` to insert the elements.
+ *              The new elements will be inserted before the element currently at this index.
+ */
+function insertElements<T>(arr: Array<T>, toInsert: Array<T>, index: number): Array<T> {
+	if(index <= 0 || index >= arr.length) {
+		throw new Error(`Index ${index} out of bounds; Min 1, max ${arr.length-1}`);
+	}
+
+	return arr.slice(0, index).concat(toInsert).concat(arr.slice(index));
+}