add new scripts to analyse results (#36)

* add new scripts to analyze results

* restore visualize.py

* add script to plot QPS vs Recall

* Fix bug

Author: StefanBogdan

.gitignore

@@ -6,3 +6,6 @@ results
 output.png
 .venv
 .env
+.ipynb_checkpoints
+*.mdx
+*.png

benchmarker/scripts/python/collate-results.py

@@ -4,35 +4,97 @@
 import os
 import glob
 import json
+from typing import List, Dict, Any
 
-def collate_results(dataset, results_directory):
-    """Collate results from the results directory into a markdown table."""
-    json_files_pattern = os.path.join(results_directory, "*.json")
 
+INSTANCE_TYPE = 'n4-highmem-16'
+RUN = "hnsw"
+IGNORE_FIRST_TEST = True
+EF_VALS = [64]
+
+def get_all_data_as_dict(results_directory: str) -> List[Dict[str,Any]]:
     data = []
 
-    for file_path in glob.glob(json_files_pattern):
+    for file_path in glob.glob(results_directory + "/*.json"):
         with open(file_path, 'r') as file:
-            file_data = json.load(file)
+            file_data = json.load(file)[(IGNORE_FIRST_TEST == True):]
             data.extend(file_data)
 
-    filtered_data = [entry for entry in data if entry['dataset_file'] == dataset]
-    sorted_data = sorted(filtered_data, key=lambda x: x['qps'], reverse=True)
+    return data
+
+
+def filter_data(data: List[Dict[str,Any]], dataset: str, limit: int, sorted_by: str="qps") -> List[Dict[str,Any]]:
+    def _filter_data(item: Dict) -> bool:
+        if (
+            item['dataset_file'] == dataset 
+            and item['limit'] == limit
+            and item['instance_type'] == INSTANCE_TYPE
+            and item['run'] == RUN
+            and item['ef'] in EF_VALS
+        ):
+            return True
+        return False
+
+    return sorted(
+        [entry for entry in data if _filter_data(entry)],
+        key=lambda x: x[sorted_by],
+        reverse=True,
+    )
+
+
+def collate_results(data: List[Dict[str,Any]], out=None):
+    """Collate results from the results directory into a markdown table."""
 
-    print("""| efConstruction | maxConnections | ef | **Recall** | **QPS** | Mean Latency | p99 Latency | Import time |
-| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |""")
-    for entry in sorted_data:
+    print("| efConstruction | maxConnections | ef | **Recall** | **QPS** | Mean Latency | p99 Latency | Import time |", file=out)
+    print("| ----- | ----- | ----- | ----- | ----- | ----- | ----- | ----- |", file=out)
+    for entry in data:
         latencyms = "{:.2f}ms".format(entry['meanLatency'] * 1000)
         p99ms = "{:.2f}ms".format(entry['p99Latency'] * 1000)
         recallfmt = "{:.2f}%".format(entry['recall'] * 100)
-        importtimefmt = "{:.2f}s".format(entry['importTime'])
+        importtimefmt = "{:.0f}s".format(entry['importTime'])
         qpsfmt = "{:.0f}".format(entry['qps'])
-        print(f"| {entry['efConstruction']} | {entry['maxConnections']} | {entry['ef']} | **{recallfmt}** | **{qpsfmt}** | {latencyms} | {p99ms} | {importtimefmt} |")
+        print(f"| {entry['efConstruction']} | {entry['maxConnections']} | {entry['ef']} | **{recallfmt}** | **{qpsfmt}** | {latencyms} | {p99ms} | {importtimefmt} |", file=out)
+
+
+def weaviate_io_results(results_directory: str):
+
+    data = get_all_data_as_dict(results_directory)
+    datasets = set()
+    limits = set()
+
+    for item in data:
+        datasets.add(item["dataset_file"])
+        limits.add(item["limit"])
+
+    for dataset in datasets:
+        with open(f"ann-{dataset.replace(".hdf5", ".mdx")}", mode="w") as file:
+            print("import Tabs from '@theme/Tabs';", file=file)
+            print("import TabItem from '@theme/TabItem';\n", file=file)
+            print('<Tabs groupId="limits">', file=file)
+            for limit in limits:
+                print(f'<TabItem value="{limit}" label="Limit {limit}">\n', file=file)
+                collate_results(
+                    data=filter_data(data, dataset, limit),
+                    out=file,
+                )
+                print('\n</TabItem>', file=file)
+            print('</Tabs>', file=file)
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Collate ann results into markdown tables.")
-    parser.add_argument('-d', '--dataset', required=True, help="The dataset file to filter by.")
+    parser.add_argument('-d', '--dataset', default="all", type=str, help="The dataset file to filter by. If the value is 'all' or not specified, it will be computed for all datasets.")
     parser.add_argument('-r', '--results', default="./results", help="The directory containing benchmark results")
+    parser.add_argument('-l', '--limit', default=10, type=int, help="The number of results returned by the ANN to fiter by")
     args = parser.parse_args()
 
-    collate_results(args.dataset, os.path.expanduser(args.results))
+    if args.dataset != "all":
+        filtered_data = filter_data(
+            data=get_all_data_as_dict(args.results),
+            dataset=args.dataset,
+            limit=args.limit,
+        )
+
+        collate_results(filtered_data)
+    else:
+        weaviate_io_results(args.results)

benchmarker/scripts/python/performance-graphs.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import json
+import argparse
+import seaborn as sns
+import matplotlib.ticker as tkr
+import matplotlib.pyplot as plt
+import pandas as pd
+
+EF_CONSTRUCTION = 256
+MAX_CONNECTIONS = 32
+RUNS = {
+    "hnsw",
+}
+
+
+def custom_filter(row):
+    # All ef values that are lowern tha the `limit` are set to the `limit`
+    if row['limit'] == 100 and row['ef'] < 90:
+        return False
+    return True 
+
+
+def get_datapoints(dataset:str, path: str):
+    datapoints = []
+    for filename in glob.glob(os.path.join(path, "*.json")):
+        with open(os.path.join(os.getcwd(), filename), "r") as f:
+            parsed = json.loads(f.read())
+            datapoints += parsed[1:]
+    df = pd.DataFrame(datapoints)
+    return df[
+        (df["dataset_file"] == dataset)             # filter for a specific dataset
+        & (df['run'].isin(RUNS))                    # remove PQ/BQ/SQ results
+        & (df["maxConnections"] == MAX_CONNECTIONS)
+        & (df["efConstruction"] == EF_CONSTRUCTION)
+        & (df.apply(custom_filter, axis=1))
+    ]
+
+
+def create_plot(results_df: pd.DataFrame):
+
+    dataset = results_df["dataset_file"].iloc[0]
+
+    sns.set_theme(
+        style='whitegrid',
+        font_scale=1.2,
+        rc={
+            # 'axes.grid': True,
+            # 'savefig.transparent': True,
+            # 'axes.facecolor': color,
+            # 'figure.facecolor': color,
+            # 'axes.edgecolor': color,
+            # 'grid.color': color,
+            # 'ytick.labelcolor': color,
+            # 'xtick.labelcolor': color,
+            }
+    )
+    plot = sns.relplot(
+        linewidth=3,
+        height=7,
+        aspect=1.5,
+        marker="o",
+        dashes=False,
+        data=results_df,
+        kind="line",
+        x="recall",
+        y="qps",
+        hue="limit",
+        style="limit",
+        palette=["b", "g"],
+    )
+    plot.set_axis_labels(
+        x_var="Recall, [%]",
+        y_var="QPS",
+    )
+    plot.figure.subplots_adjust(top=0.85)
+    plot.figure.suptitle(
+        f"Query Performance, {dataset}",
+        weight="bold",
+        
+    )
+    sns.move_legend(
+        plot,
+        "lower center",
+        bbox_to_anchor=(.5, .84),
+        ncol=3,
+        title="Limit: ",
+        frameon=False,
+    )
+
+
+    plot.axes[0][0].get_xaxis().set_major_formatter(tkr.FuncFormatter(lambda x, _: f'{x*100:.0f}'))
+    plot.axes[0][0].get_yaxis().set_major_formatter(tkr.StrMethodFormatter('{x:,.0f}'))
+    plt.savefig(f"{dataset.split('.')[0]}.png", bbox_inches='tight')
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Collate ann results into markdown tables.")
+    parser.add_argument('-d', '--dataset', required=True, help="The dataset file to filter by")
+    parser.add_argument('-r', '--results', default="./results", help="The directory containing benchmark results")
+    args = parser.parse_args()
+
+    create_plot(
+        get_datapoints(args.dataset, os.path.expanduser(args.results)),
+    )