Stream hyperopt-result in small batches

Avoiding memory-exhaustion on huge hyperopt results

closes #5305
closes #5149

tests/optimize/test_hyperopt_tools.py

@@ -20,9 +20,14 @@ def create_results() -> List[Dict]:
 
 
 def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None:
+
+    hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt')
+
+    hyperopt_epochs = HyperoptTools.load_filtered_results(hyperopt.results_file, {})
+    assert hyperopt_epochs == ([], 0)
+
     # Test writing to temp dir and reading again
     epochs = create_results()
-    hyperopt.results_file = Path(tmpdir / 'ut_results.fthypt')
 
     caplog.set_level(logging.DEBUG)
 
@@ -33,15 +38,28 @@ def test_save_results_saves_epochs(hyperopt, tmpdir, caplog) -> None:
     hyperopt._save_result(epochs[0])
     assert log_has(f"2 epochs saved to '{hyperopt.results_file}'.", caplog)
 
-    hyperopt_epochs = HyperoptTools.load_previous_results(hyperopt.results_file)
+    hyperopt_epochs = HyperoptTools.load_filtered_results(hyperopt.results_file, {})
     assert len(hyperopt_epochs) == 2
+    assert hyperopt_epochs[1] == 2
+    assert len(hyperopt_epochs[0]) == 2
+
+    result_gen = HyperoptTools._read_results(hyperopt.results_file, 1)
+    epoch = next(result_gen)
+    assert len(epoch) == 1
+    assert epoch[0] == epochs[0]
+    epoch = next(result_gen)
+    assert len(epoch) == 1
+    epoch = next(result_gen)
+    assert len(epoch) == 0
+    with pytest.raises(StopIteration):
+        next(result_gen)
 
 
 def test_load_previous_results2(mocker, testdatadir, caplog) -> None:
     results_file = testdatadir / 'hyperopt_results_SampleStrategy.pickle'
     with pytest.raises(OperationalException,
                        match=r"Legacy hyperopt results are no longer supported.*"):
-        HyperoptTools.load_previous_results(results_file)
+        HyperoptTools.load_filtered_results(results_file, {})
 
 
 @pytest.mark.parametrize("spaces, expected_results", [