Merge pull request #42 from clue-labs/robots

Add `robots.txt`

Author: clue

Makefile

@@ -6,7 +6,7 @@ build: public/src/tailwind.min.css
 	cp overrides/* source/overrides/
 	docker run --rm -i -v ${PWD}/source:/docs -u $(shell id -u) squidfunk/mkdocs-material:8.1.3 build
 	cp -r source/build/docs/ build/ && rm build/docs/sitemap.xml.gz
-	cp public/.htaccess public/index.html build/
+	cp public/.htaccess public/index.html public/robots.txt build/
 	cp public/src/* build/src/
 
 public/src/tailwind.min.css: public/index.html tailwindcss

public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow:

tests/integration.bash

@@ -33,6 +33,10 @@ match -iP "Content-Type: text/html[\r\n]"
 match -iP "Content-Encoding: gzip[\r\n]"
 match -iP "Vary: Accept-Encoding[\r\n]"
 
+curl -v $base/robots.txt
+match "HTTP/.* 200"
+match -iP "Content-Type: text/plain[;\r\n]"
+
 curl -v $base/invalid
 match "HTTP/.* 404"
 match -i "Content-Type: text/html"