From c1e124d94c6524509701b65cb7d1f66848510190 Mon Sep 17 00:00:00 2001
From: MagicLex <64143547+MagicLex@users.noreply.github.com>
Date: Tue, 1 Apr 2025 13:35:22 +0300
Subject: [PATCH] Update robots.txt

changed the principle to disallow everything BUT latest release. Will need to be updated every release.
---
 docs/robots.txt | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/docs/robots.txt b/docs/robots.txt
index a96e321af..e7b0626cd 100644
--- a/docs/robots.txt
+++ b/docs/robots.txt
@@ -1,4 +1,10 @@
 User-agent: *
-Disallow:
+Disallow: /
+Allow: /4.1/
+Allow: /latest/
 
-Sitemap: https://docs.hopsworks.ai/3.2/sitemap.xml
\ No newline at end of file
+Allow: /hopsworks-api/4.1/
+Allow: /hopsworks-api/latest/
+
+Allow: /hopsworks-api/4.1/javadoc/
+Allow: /hopsworks-api/latest/javadoc/