Use robots.txt and mitigate AI bots

html/robots.txt

@@ -1,6 +1,16 @@
-User-agent: SiteimproveBot
-Disallow: /versions
-Disallow: /old_site
-
+# General bots
+# Note: Google does not respect Crawl-delay, but that's fine.
 User-agent: *
-Allow: /
+Allow: /project/*/users/*
+Disallow: /project/
+Disallow: /api/
+Crawl-delay: 10
+
+# AI crawlers — slower, same path rules duplicated
+User-agent: GPTBot
+User-agent: ClaudeBot
+User-agent: PerplexityBot
+Allow: /project/*/users/*
+Disallow: /project/
+Disallow: /api/
+Crawl-delay: 30

models/projects.lua

@@ -117,7 +117,7 @@ local ActiveProjects = Model:extend('active_projects', {
                 '#present:Username=' .. escape(self.username) ..
                 '&ProjectName=' .. escape(self.projectname) ..
                 '&editMode&noRun',
-            download = '/project/' .. escape(self.id),
+            download = '/api/v1/project/' .. escape(self.id),
             site = '/project?username=' .. escape(self.username) ..
                 '&projectname=' .. escape(self.projectname),
             author = '/user?username=' .. escape(self.username),

nginx.conf

@@ -46,6 +46,7 @@ http {
 
     include nginx.conf.d/logging.conf;
     include nginx.conf.d/mime.types;
+    include nginx.conf.d/snap-bot-mitigation.conf;
 
     resolver ${{DNS_RESOLVER}};
 

nginx.conf.d/locations.conf

@@ -1,6 +1,10 @@
 # Shared location configurations for all environments
 # These are included in both the main SSL server block (prod) and non-SSL server block (dev).
 
+# Bot mitigation (robots.txt + UA blocklist). Included here so it is applied
+# to every server block — both prod hosts, both staging hosts, and dev.
+include nginx.conf.d/snap-bot-mitigation-server.conf;
+
 # Specify the cloud domain each page should use.
 
 set_by_lua_block $cloud_url { return os.getenv('CLOUD_URL') }
@@ -48,6 +52,19 @@ location @lapisapp {
     }
 }
 
+# Raw project XML at /project/<numeric_id> — Lapis-served and the largest
+# crawler egress source, so it gets rate-limited via the snap_project zone
+# (defined in snap-bot-mitigation.conf). The upcoming HTML route
+# /project/<name>/users/<user> does not match this regex and is unaffected.
+location ~ ^/project/[0-9]+/?$ {
+    limit_req zone=snap_project burst=120 nodelay;
+    access_log logs/lapis_access.log main_ext if=$should_log;
+    default_type text/html;
+    content_by_lua_block {
+        require("lapis").serve("app")
+    }
+}
+
 # Static content from snapCloud
 # Avoid unnecessarily logging data for CSS, JS, etc.
 location /static/ {

nginx.conf.d/snap-bot-mitigation-server.conf

@@ -0,0 +1,18 @@
+# Bot-mitigation directives that live in server{} context. Included from
+# locations.conf so every server block (prod + staging, both hosts, plus
+# dev) picks them up. The http-context map and limit_req_zone live in
+# nginx.conf.d/snap-bot-mitigation.conf.
+
+# Serve robots.txt as a static file from the repo. Host-independent — does
+# not depend on the per-host static root or the Lapis app.
+location = /robots.txt {
+    access_log off;
+    default_type text/plain;
+    root html;
+}
+
+# 403 any user-agent flagged by $snap_block_ua. `return` is one of the few
+# directives that is safe to use inside `if`.
+if ($snap_block_ua) {
+    return 403;
+}

nginx.conf.d/snap-bot-mitigation.conf

@@ -0,0 +1,24 @@
+# Bot-mitigation directives that must live in the http{} context. The
+# matching server-context directives are in snap-bot-mitigation-server.conf,
+# included from locations.conf.
+
+# UA blocklist for SEO crawlers that ignore robots.txt. \b word-boundaries
+# guard against substring false positives (e.g. "yeti" inside a longer UA).
+map $http_user_agent $snap_block_ua {
+    default          0;
+    "~*ahrefsbot"    1;
+    "~*semrushbot"   1;
+    "~*\bdotbot\b"   1;
+    "~*mj12bot"      1;
+    "~*sleepbot"     1;
+    "~*\byeti\b"     1;
+    "~*blexbot"      1;
+    "~*petalbot"     1;
+}
+
+# Rate-limit zone for the raw XML at /project/<numeric_id>. The rate is
+# deliberately generous because Snap! is used in classrooms where 20–40
+# students share one NAT IP; tighten only with evidence from error.log
+# ("limiting requests" lines).
+limit_req_zone $binary_remote_addr zone=snap_project:10m rate=60r/s;
+limit_req_status 429;

views/partials/flag_list.etlua

@@ -43,7 +43,7 @@
             'Are you sure you want to remove this flag?',
             () => {
                 cloud.delete(
-                    '/project/<%= project.id %>/flag',
+                    '/api/v1/project/<%= project.id %>/flag',
                     null,
                     { flagger: flagger }
                 );
@@ -56,7 +56,7 @@
                 'report the flagger for abusing the flagging system?',
             () => {
                 cloud.delete(
-                    '/project/<%= project.id %>/flag',
+                    '/api/v1/project/<%= project.id %>/flag',
                     null,
                     { flagger: flagger, report: true }
                 );

views/partials/project_buttons.etlua

@@ -10,14 +10,14 @@
                 aria-label="<%= locale.get('unbookmark') %>"
                 title="<%= locale.get('unbookmark') %>"
                 onclick="cloud.delete(
-                '/project/<%= project.id %>/bookmark/<%= current_user.id %>')"
+                '/api/v1/project/<%= project.id %>/bookmark/<%= current_user.id %>')"
             ><i class="fas fa-heart"></i></button>
         <% else %>
             <button class="btn btn-outline-dark bookmark" type="button"
                 aria-label="<%= locale.get('bookmark') %>"
                 title="<%= locale.get('bookmark') %>"
                 onclick="cloud.post(
-                '/project/<%= project.id %>/bookmark/<%= current_user.id %>')"
+                '/api/v1/project/<%= project.id %>/bookmark/<%= current_user.id %>')"
             ><i class="far fa-heart"></i></button>
         <% end %>
     <% end %>
@@ -28,7 +28,8 @@
     %></a>
     <a class="btn btn-outline-primary download"
         aria-label="<%= locale.get('download')%> Project XML"
-        href="<%= project:url_for('download', params.devVersion) %>" download
+        href="<%= project:url_for('download', params.devVersion) %>"
+        download="<%= project.projectname %>.xml"
         ><%= locale.get('download') %></a>
     <button class="btn btn-outline-primary embed-button" type="button"
         onclick="embedDialog()"><%= locale.get('embed') %></button>
@@ -128,7 +129,7 @@ function confirmDelete () {
         '<%- package.loaded.dialog(
             'confirm_delete',
             { item_name = 'project'}) %>',
-        () => { cloud.delete('/project/<%= project.id %>'); }
+        () => { cloud.delete('/api/v1/project/<%= project.id %>'); }
     );
 }
 
@@ -143,7 +144,7 @@ function confirmFlag () {
                     var form =
                         document.querySelector('form.reasons');
                     cloud.post(
-                        '/project/<%= project.id %>/flag',
+                        '/api/v1/project/<%= project.id %>/flag',
                         null,
                         {
                             reason: form.querySelector(
@@ -161,7 +162,7 @@ function confirmFlag () {
 }
 
 function unflagProject() {
-    cloud.delete('/project/<%= project.id %>/flag')
+    cloud.delete('/api/v1/project/<%= project.id %>/flag')
 }
 
 function markAsRemix () {
@@ -170,7 +171,7 @@ function markAsRemix () {
         input => {
             var url = new URL(input);
             cloud.post(
-                '/project/<%= project.id %>/mark_as_remix',
+                '/api/v1/project/<%= project.id %>/mark_as_remix',
                 null,
                 {
                     username: '<%= project.username %>',

views/partials/project_details.etlua

@@ -42,34 +42,34 @@
         <button class="btn btn-secondary" type="button"
             data-confirmation-message="<%= locale.get('confirm_unpublish_project') %>"
             data-confirmation-method="delete"
-            data-confirmation-url="/project/<%= project.id %>/publish"
+            data-confirmation-url="/api/v1/project/<%= project.id %>/publish"
         ><%= locale.get('unpublish_button') %></button>
         <button class="btn btn-secondary" type="button"
             data-confirmation-message="<%= locale.get('confirm_unshare_project') %>"
             data-confirmation-method="delete"
-            data-confirmation-url="/project/<%= project.id %>/share"
+            data-confirmation-url="/api/v1/project/<%= project.id %>/share"
         ><%= locale.get('unshare_button') %></button>
         <% elseif project.ispublic then %>
         <button class="btn btn-secondary publish" type="button"
             data-confirmation-message="<%= locale.get('confirm_publish_project') %>"
             data-confirmation-method="post"
-            data-confirmation-url="/project/<%= project.id %>/publish"
+            data-confirmation-url="/api/v1/project/<%= project.id %>/publish"
         ><%= locale.get('publish_button') %></button>
         <button class="btn btn-secondary unshare" type="button"
             data-confirmation-message="<%= locale.get('confirm_unshare_project') %>"
             data-confirmation-method="delete"
-            data-confirmation-url="/project/<%= project.id %>/share"
+            data-confirmation-url="/api/v1/project/<%= project.id %>/share"
         ><%= locale.get('unshare_button') %></button>
         <% else %>
         <button class="btn btn-secondary share" type="button"
             data-confirmation-message="<%= locale.get('confirm_share_project') %>"
             data-confirmation-method="post"
-            data-confirmation-url="/project/<%= project.id %>/share"
+            data-confirmation-url="/api/v1/project/<%= project.id %>/share"
         ><%= locale.get('share_button') %></button>
         <button class="btn btn-secondary" type="button"
             data-confirmation-message="<%= locale.get('confirm_publish_project') %>"
             data-confirmation-method="post"
-            data-confirmation-url="/project/<%= project.id %>/publish"
+            data-confirmation-url="/api/v1/project/<%= project.id %>/publish"
         ><%= locale.get('publish_button') %></button>
         <% end %>
     <% end %>

views/static/resources.lua

@@ -40,7 +40,7 @@ local materials = {
   {
     title = "Get Coding with Snap<em>!</em>",
     author = 'openSAP',
-    url = "https://open.sap.com/courses/snap1",
+    url = "https://learning.sap.com/learning-journeys/get-coding-with-snap-building-up-to-ai",
     language = {"English"},
     type = "course",
     level = 'beginner',
@@ -51,7 +51,7 @@ local materials = {
   {
     title = "From Media Computation to Data Science",
     author = 'openSAP',
-    url = "https://open.sap.com/courses/snap2",
+    url = "https://learning.sap.com/courses/from-media-computation-to-data-science",
     language = {"English"},
     type = "course",
     level = nil,
@@ -70,7 +70,7 @@ local materials = {
     description = nil,
     image = nil
   },
-    {
+  {
     title = "BJC Sparks",
     author = 'UC Berkeley and EDC',
     url = "https://bjc.berkeley.edu/sparks/",