Add smoke test GitHub Action workflow (#379)

Author: dbarbuzzi

.github/workflows/local-install-check.yaml

@@ -1,5 +1,5 @@
 name: Local Install Checks
-on: 
+on:
   push:
     branches:
       - main
@@ -12,6 +12,9 @@ on:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
+env:
+  SPARSEZOO_TEST_MODE: "true"
+
 jobs:
   local-install-test:
     runs-on: ubuntu-20.04

.github/workflows/test-check.yaml

@@ -0,0 +1,43 @@
+name: Test Checks
+
+on:
+  push:
+    branches:
+      - main
+      - 'release/*'
+  pull_request:
+    branches:
+      - main
+      - 'release/*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  SPARSEZOO_TEST_MODE: true
+
+jobs:
+  base-tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: ⚙️ Install dependencies
+        run: pip3 install .[dev]
+      - name: Run base tests
+        run: make test
+  cli-smoke-tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: ⚙️ Install dependencies
+        run: pip3 install .[dev,server]
+      - name: Run CLI smoke tests
+        run: PYTEST_ARGS="-m smoke" make test TARGETS=cli,nobase
+  examples-smoke-tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install dependencies
+        run: pip3 install .[dev]
+      - name: Run examples smoke tests
+        run: PYTEST_ARGS="-m smoke" make test TARGETS=examples,nobase

Makefile

@@ -9,16 +9,20 @@ MDCHECKGLOBS := 'docs/**/*.md' 'docs/**/*.rst' 'examples/**/*.md' 'scripts/**/*.
 MDCHECKFILES := CODE_OF_CONDUCT.md CONTRIBUTING.md DEVELOPING.md README.md
 SPARSEZOO_TEST_MODE := "true"
 
-TARGETS := ""  # targets for running pytests: cli,examples
+TARGETS := ""  # targets for running pytests: cli,examples,nobase
 PYTEST_ARGS ?= ""
 ifneq ($(findstring cli,$(TARGETS)),cli)
-	PYTEST_ARGS := $(PYTEST_ARGS) --ignore tests/test_benchmark.py \
+	PYTEST_ARGS += --ignore tests/test_benchmark.py \
 	--ignore tests/test_check_hardware.py \
 	--ignore tests/test_run_inference.py \
 	--ignore tests/test_server.py
 endif
 ifneq ($(findstring examples,$(TARGETS)),examples)
-	PYTEST_ARGS := $(PYTEST_ARGS) --ignore tests/test_examples.py
+	PYTEST_ARGS += --ignore tests/test_examples.py
+endif
+ifeq ($(findstring nobase,$(TARGETS)),nobase)
+	PYTEST_ARGS += --ignore tests/utils/test_data.py \
+	--ignore tests/test_engine.py
 endif
 
 PYTHON := python3

pyproject.toml

@@ -1,3 +1,8 @@
 [tool.black]
 line-length = 88
 target-version = ['py36']
+
+[tool.pytest.ini_options]
+markers = [
+    "smoke: smoke tests",
+]

tests/deepsparse/image_classification/test_pipelines.py

@@ -14,8 +14,6 @@
 
 
 import numpy
-from PIL import Image
-from torchvision import transforms
 
 import pytest
 from deepsparse import Pipeline
@@ -27,6 +25,10 @@
 from sparsezoo.utils import load_numpy_list
 
 
+from PIL import Image  # isort:skip
+from torchvision import transforms  # isort:skip
+
+
 @pytest.mark.parametrize(
     "zoo_stub,image_size,num_samples",
     [
@@ -38,6 +40,7 @@
         )
     ],
 )
+@pytest.mark.smoke
 def test_image_classification_pipeline_preprocessing(zoo_stub, image_size, num_samples):
     non_rand_resize_scale = 256.0 / 224.0  # standard used
     standard_imagenet_transforms = transforms.Compose(

tests/test_benchmark.py

@@ -18,6 +18,7 @@
 from helpers import predownload_stub, run_command
 
 
+@pytest.mark.smoke
 def test_benchmark_help():
     cmd = ["deepsparse.benchmark", "--help"]
     print(f"\n==== test_benchmark_help command ====\n{' '.join(cmd)}")
@@ -52,10 +53,11 @@ def test_benchmark_help():
             "bookcorpus_wikitext/base-none",
             ["-t", "20"],
         ),
-        (
+        pytest.param(
             "zoo:nlp/masked_language_modeling/bert-base/pytorch/huggingface/"
             "bookcorpus_wikitext/12layer_pruned90-none",
             [],
+            marks=pytest.mark.smoke,
         ),
         (
             "zoo:cv/classification/resnet_v1-50/pytorch/sparseml/imagenet/base-none",

tests/test_check_hardware.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import pytest
 from helpers import run_command
 
 
+@pytest.mark.smoke
 def test_check_hardware():
     cmd = ["deepsparse.check_hardware"]
     print(f"\n==== deepsparse.check_hardware command ====\n{' '.join(cmd)}")

tests/test_engine.py

@@ -40,6 +40,7 @@
         ]
     ),
 )
+@pytest.mark.smoke
 class TestEngineParametrized:
     def test_engine(self, model: Model, batch_size: int):
         """

tests/test_examples.py

@@ -46,11 +46,9 @@
     "model, batch_size",
     (
         [
-            pytest.param(
-                mobilenet_v1,
-                b,
-            )
-            for b in [1, 8, 64]
+            pytest.param(mobilenet_v1, 1, marks=pytest.mark.smoke),
+            pytest.param(mobilenet_v1, 8, marks=pytest.mark.smoke),
+            pytest.param(mobilenet_v1, 64),
         ]
     ),
 )
@@ -70,11 +68,9 @@ def test_check_correctness(model: Model, batch_size: int):
     "model, batch_size",
     (
         [
-            pytest.param(
-                mobilenet_v1,
-                b,
-            )
-            for b in [1, 8, 64]
+            pytest.param(mobilenet_v1, 1, marks=pytest.mark.smoke),
+            pytest.param(mobilenet_v1, 8, marks=pytest.mark.smoke),
+            pytest.param(mobilenet_v1, 64),
         ]
     ),
 )
@@ -94,11 +90,9 @@ def test_run_benchmark(model: Model, batch_size: int):
     "model_name, batch_size",
     (
         [
-            pytest.param(
-                "mobilenet_v1",
-                b,
-            )
-            for b in [1, 8, 64]
+            pytest.param("mobilenet_v1", 1, marks=pytest.mark.smoke),
+            pytest.param("mobilenet_v1", 8, marks=pytest.mark.smoke),
+            pytest.param("mobilenet_v1", 64),
         ]
     ),
 )
@@ -117,11 +111,9 @@ def test_classification(model_name: str, batch_size: int):
     "model_name, batch_size",
     (
         [
-            pytest.param(
-                "yolo_v3",
-                b,
-            )
-            for b in [1, 8, 64]
+            pytest.param("yolo_v3", 1, marks=pytest.mark.smoke),
+            pytest.param("yolo_v3", 8, marks=pytest.mark.smoke),
+            pytest.param("yolo_v3", 64),
         ]
     ),
 )

tests/test_run_inference.py

@@ -21,6 +21,7 @@
 from helpers import predownload_stub, run_command
 
 
+@pytest.mark.smoke
 def test_run_inference_help():
     cmd = ["deepsparse.transformers.run_inference", "--help"]
     print(f"\n==== test_run_inference_help command ====\n{' '.join(cmd)}")
@@ -33,6 +34,7 @@ def test_run_inference_help():
     assert "fail" not in res.stdout.lower()
 
 
+@pytest.mark.smoke
 def test_run_inference_ner(cleanup: Dict[str, List]):
     cmd = [
         "deepsparse.transformers.run_inference",
@@ -69,11 +71,12 @@ def test_run_inference_ner(cleanup: Dict[str, List]):
 @pytest.mark.parametrize(
     ("input_format", "model_path", "local_model"),
     [
-        (
+        pytest.param(
             "csv",
             "zoo:nlp/question_answering/bert-base/pytorch/huggingface/squad/"
             "pruned_6layers-aggressive_98",
             True,
+            marks=pytest.mark.smoke,
         ),
         (
             "json",
@@ -138,11 +141,12 @@ def test_run_inference_qa(
             True,
             ["--num-cores", "4", "--engine-type", "onnxruntime"],
         ),
-        (
+        pytest.param(
             "csv",
             "zoo:nlp/text_classification/bert-base/pytorch/huggingface/sst2/base-none",
             True,
             [],
+            marks=pytest.mark.smoke,
         ),
         (
             "json",

tests/test_server.py

@@ -17,6 +17,7 @@
 
 import requests
 
+import pytest
 from helpers import predownload_stub, run_command, wait_for_server
 
 
@@ -25,6 +26,7 @@
 # TODO: Include additional opts/etc. in tests
 
 
+@pytest.mark.smoke
 def test_server_help():
     cmd = ["deepsparse.server", "--help"]
     print(f"\n==== test_server_help command ====\n{' '.join(cmd)}\n==== ====")
@@ -84,6 +86,7 @@ def test_server_ner(cleanup: Dict[str, List]):
     assert "fail" not in output.lower()
 
 
+@pytest.mark.smoke
 def test_server_qa(cleanup: Dict[str, List]):
     cmd = [
         "deepsparse.server",

tests/utils/test_data.py

@@ -30,6 +30,7 @@
         [numpy.random.randint(255, size=(3, 244, 244), dtype=numpy.uint8)],
     ),
 )
+@pytest.mark.smoke
 def test_arrays_bytes_conversion(arrays):
     serialized_arrays = arrays_to_bytes(arrays)
     assert isinstance(serialized_arrays, bytearray)