add shell and python scripts directories, add update stability test

Author: StefanBogdan

benchmarker/Dockerfile

@@ -1,5 +1,5 @@
 FROM golang:alpine
-RUN apk add --no-cache hdf5-dev gcc libc-dev python3
+RUN apk add --no-cache hdf5-dev gcc libc-dev python3 bash
 WORKDIR /app
 COPY . .
 RUN CGO_ENABLED=1 go build -o benchmarker .

benchmarker/scripts/ann.py renamed to benchmarker/scripts/python/ann.py

@@ -0,0 +1,39 @@
+import os
+import glob
+import json
+import unittest
+
+
+PATH = "./results"
+REQUIRED_RECALL = .992
+
+
+class TestResults(unittest.TestCase):
+
+    def setUp(self):
+        self.datapoints = []
+
+        for result_filename in glob.glob(os.path.join(PATH, "*.json")):
+            with open(os.path.join(os.getcwd(), result_filename), "r") as result_file:
+                self.datapoints.append(json.load(result_file))
+
+    def test_max_recall(self):
+
+        rr_env = os.getenv("REQUIRED_RECALL")
+
+        if rr_env:
+            required_recall = float(rr_env)
+        else:
+            required_recall = REQUIRED_RECALL
+
+        for run_iteration in self.datapoints:
+
+            max_recall = max([run_config["recall"] for run_config in run_iteration])
+            self.assertTrue(
+                max_recall >= required_recall,
+                f"need to achieve at least {required_recall} recall, got only {max_recall}",
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

benchmarker/scripts/collate-results.py renamed to benchmarker/scripts/python/collate-results.py

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -eou pipefail
+
+echo "Run benchmark script"
+/app/benchmarker ann-benchmark \
+  -v /app/datasets/${DATASET}.hdf5 \
+  --distance $DISTANCE \
+  --indexType $INDEX_TYPE \
+  --updatePercentage $UPDATE_PERCENTAGE \
+  --cleanupIntervalSeconds $CLEANUP_INTERVAL_SECONDS \
+  --updateIterations $UPDATE_ITERATIONS \
+  --grpcOrigin "${WEAVIATE_URL}:50051" \
+  --httpOrigin "${WEAVIATE_URL}:8080" \
+  --updateRandomized
+
+
+echo "Run complete, now analyze the results"
+python3 /app/scripts/python/update_stability.py
+
+echo "Passed!"