Add robots.txt and sitemap.xml for search engines

lib/microwaveprop_web.ex

@@ -17,7 +17,7 @@ defmodule MicrowavepropWeb do
   those modules here.
   """
 
-  def static_paths, do: ~w(assets fonts images downloads favicon.ico robots.txt)
+  def static_paths, do: ~w(assets fonts images downloads favicon.ico robots.txt sitemap.xml)
 
   def router do
     quote do

priv/static/robots.txt

@@ -1,5 +1,8 @@
-# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /
+User-agent: *
+Allow: /
+Disallow: /backfill
+Disallow: /dev/
+Disallow: /dashboard
+Disallow: /health
+
+Sitemap: https://prop.w5isp.com/sitemap.xml

priv/static/sitemap.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://prop.w5isp.com/</loc>
+    <changefreq>daily</changefreq>
+    <priority>1.0</priority>
+  </url>
+  <url>
+    <loc>https://prop.w5isp.com/map</loc>
+    <changefreq>hourly</changefreq>
+    <priority>0.9</priority>
+  </url>
+  <url>
+    <loc>https://prop.w5isp.com/submit</loc>
+    <changefreq>monthly</changefreq>
+    <priority>0.7</priority>
+  </url>
+  <url>
+    <loc>https://prop.w5isp.com/contacts</loc>
+    <changefreq>daily</changefreq>
+    <priority>0.6</priority>
+  </url>
+  <url>
+    <loc>https://prop.w5isp.com/algo</loc>
+    <changefreq>monthly</changefreq>
+    <priority>0.5</priority>
+  </url>
+</urlset>