Added robots.txt

Cargo.toml

@@ -9,6 +9,8 @@ edition = "2021"
 anyhow = "1.0.75"
 askama = "0.12.1"
 axum = "0.6"
+serde = { version = "1.0.197", features = ["derive"] }
+surrealdb = "1.3.1"
 tokio = { version = "1.35.0", features = ["full"] }
 tower = "0.4.13"
 tower-http = { version = "0.4.4", features = ["fs"] }

assets/robots.txt

@@ -0,0 +1,31 @@
+User-agent: *
+Disallow:
+
+User-agent: AdsBot-Google
+User-agent: Amazonbot
+User-agent: anthropic-ai
+User-agent: Applebot
+User-agent: AwarioRssBot
+User-agent: AwarioSmartBot
+User-agent: Bytespider
+User-agent: CCBot
+User-agent: ChatGPT-User
+User-agent: ClaudeBot
+User-agent: Claude-Web
+User-agent: cohere-ai
+User-agent: DataForSeoBot
+User-agent: FacebookBot
+User-agent: Google-Extended
+User-agent: GoogleOther
+User-agent: GPTBot
+User-agent: ImagesiftBot
+User-agent: magpie-crawler
+User-agent: Meltwater
+User-agent: omgili
+User-agent: omgilibot
+User-agent: peer39_crawler
+User-agent: peer39_crawler/1.0
+User-agent: PerplexityBot
+User-agent: Seekr
+User-agent: YouBot
+Disallow: /

src/html/root.rs

@@ -1,7 +1,9 @@
 use crate::html;
 use askama::Template;
-use axum::response::IntoResponse;
-use axum::{routing::get, Router};
+use axum::{
+    response::{IntoResponse, Redirect},
+    routing::{get, Router},
+};
 use tower_http::services::ServeDir;
 
 pub fn get_router() -> Router {
@@ -14,6 +16,10 @@ pub fn get_router() -> Router {
         .route("/now", get(now))
         .route("/about", get(about))
         .route("/contact", get(contact))
+        .route(
+            "/robots.txt",
+            get(|| async { Redirect::permanent("/assets/robots.txt") }),
+        )
         .nest_service(
             "/assets",
             ServeDir::new(format!("{}/assets", assets_path.to_str().unwrap())),
@@ -22,8 +28,8 @@ pub fn get_router() -> Router {
 
 async fn home() -> impl IntoResponse {
     let template = HomeTemplate {
-		active_navbar: html::NavBar::HOME,
-	};
+        active_navbar: html::NavBar::HOME,
+    };
     html::HtmlTemplate(template)
 }
 
@@ -35,8 +41,8 @@ struct HomeTemplate {
 
 pub async fn blog() -> impl IntoResponse {
     let template = BlogTemplate {
-		active_navbar: html::NavBar::BLOG
-	};
+        active_navbar: html::NavBar::BLOG,
+    };
     html::HtmlTemplate(template)
 }
 
@@ -48,8 +54,8 @@ struct BlogTemplate {
 
 pub async fn projects() -> impl IntoResponse {
     let template = ProjectsTemplate {
-		active_navbar: html::NavBar::PROJECTS,
-	};
+        active_navbar: html::NavBar::PROJECTS,
+    };
     html::HtmlTemplate(template)
 }
 
@@ -61,8 +67,8 @@ struct ProjectsTemplate {
 
 async fn now() -> impl IntoResponse {
     let template = NowTemplate {
-		active_navbar: html::NavBar::NOW,
-	};
+        active_navbar: html::NavBar::NOW,
+    };
     html::HtmlTemplate(template)
 }
 
@@ -74,8 +80,8 @@ struct NowTemplate {
 
 async fn about() -> impl IntoResponse {
     let template = AboutTemplate {
-		active_navbar: html::NavBar::ABOUT,
-	};
+        active_navbar: html::NavBar::ABOUT,
+    };
     html::HtmlTemplate(template)
 }
 
@@ -87,8 +93,8 @@ struct AboutTemplate {
 
 async fn contact() -> impl IntoResponse {
     let template = ContactTemplate {
-		active_navbar: html::NavBar::CONTACT
-	};
+        active_navbar: html::NavBar::CONTACT,
+    };
     html::HtmlTemplate(template)
 }