feat: enhance SEO and sitemap functionality

- Updated the sitemap generation to return a byte array instead of a string for improved performance.
- Added new SEO metadata for categories and store chooser routes to enhance search engine visibility.
- Implemented redirects for legacy wishlist URLs to the homepage, improving user experience.
- Updated robots.txt to disallow indexing of specific paths, ensuring better control over search engine crawling.

These changes significantly improve the SEO capabilities and sitemap handling of the application, enhancing both performance and user navigation.

src/FrontOffice.Main/Program.cs

@@ -60,10 +60,14 @@ webApp.MapBlazorHub();
 
 webApp.MapGet("/sitemap.xml", async (SitemapGenerator sitemapGenerator, CancellationToken cancellationToken) =>
 {
-    var xml = await sitemapGenerator.GenerateAsync(cancellationToken);
+    var bytes = await sitemapGenerator.GenerateBytesAsync(cancellationToken);
-    return Results.Content(xml, "application/xml; charset=utf-8");
+    return Results.File(bytes, "application/xml; charset=utf-8");
 });
 
+// Legacy URLs from old site / common e-commerce paths — redirect to homepage
+webApp.MapGet("/wishlist", () => Results.Redirect("/", permanent: true));
+webApp.MapGet("/wishlist/", () => Results.Redirect("/", permanent: true));
+
 webApp.MapFallbackToPage("/_Host");
 
 

src/FrontOffice.Main/Utilities/Seo/SeoMetadataProvider.cs

@@ -1,3 +1,4 @@
+using System.Text.RegularExpressions;
 using Microsoft.Extensions.Options;
 
 namespace FrontOffice.Main.Utilities.Seo;
@@ -30,6 +31,9 @@ public class SeoMetadataProvider
         [RouteConstants.Store.Products] = (
             "محصولات | کارابازار سلامت",
             "فروشگاه محصولات سلامت کارابازار — خرید آنلاین با پاداش باشگاه مشتریان."),
+        [RouteConstants.Store.Categories] = (
+            "دسته‌بندی‌ها | کارابازار سلامت",
+            "دسته‌بندی محصولات فروشگاه کارابازار سلامت."),
         [RouteConstants.Package.List] = (
             "پکیج‌ها | کارابازار سلامت",
             "پکیج‌های عضویت باشگاه مشتریان کارابازار سلامت."),
@@ -45,6 +49,18 @@ public class SeoMetadataProvider
         [RouteConstants.Registration.Wizard] = (
             "ثبت‌نام | کارابازار سلامت",
             "ثبت‌نام سریع در باشگاه مشتریان کارابازار سلامت."),
+        [RouteConstants.Gateway.StoreChooser] = (
+            "فروشگاه‌ها | کارابازار سلامت",
+            "انتخاب فروشگاه اصلی یا اعتباری در کارابازار سلامت."),
+    };
+
+    private static readonly HashSet<string> NoIndexPaths = new(StringComparer.OrdinalIgnoreCase)
+    {
+        "/wishlist",
+        "/wp-admin",
+        "/wp-login.php",
+        "/feed",
+        "/xmlrpc.php",
     };
 
     private static readonly HashSet<string> NoIndexPrefixes = new(StringComparer.OrdinalIgnoreCase)
@@ -62,8 +78,20 @@ public class SeoMetadataProvider
         "/discount-store/order/",
         RouteConstants.Gateway.OrdersChooser,
         RouteConstants.Gateway.CartChooser,
+        RouteConstants.Commission.Dashboard,
+        "/commission/",
+        RouteConstants.Network.Statistics,
+        "/network/",
+        RouteConstants.Package.MyPackages,
+        "/wp-content/",
+        "/wp-includes/",
     };
 
+    private static readonly Regex ProductDetailPattern = new(@"^/product/\d+$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+    private static readonly Regex DiscountProductPattern = new(@"^/discount-store/product/\d+$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+    private static readonly Regex PackageDetailPattern = new(@"^/package/\d+$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+    private static readonly Regex CheckoutPackagePattern = new(@"^/checkout/\d+$", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant);
+
     public SeoMetadataProvider(BlogPostService blogPostService, IOptions<SeoSettings> settings)
     {
         _blogPostService = blogPostService;
@@ -121,14 +149,31 @@ public class SeoMetadataProvider
             };
         }
 
+        if (IsIndexableDynamicRoute(normalizedPath))
+        {
+            return new SeoPageMetadata
+            {
+                Title = _settings.SiteName,
+                Description = _settings.DefaultDescription,
+                CanonicalUrl = $"{baseUrl}{normalizedPath}"
+            };
+        }
+
         return new SeoPageMetadata
         {
             Title = _settings.SiteName,
             Description = _settings.DefaultDescription,
-            CanonicalUrl = $"{baseUrl}{normalizedPath}"
+            CanonicalUrl = $"{baseUrl}{normalizedPath}",
+            NoIndex = true
         };
     }
 
+    private static bool IsIndexableDynamicRoute(string path) =>
+        ProductDetailPattern.IsMatch(path)
+        || DiscountProductPattern.IsMatch(path)
+        || PackageDetailPattern.IsMatch(path)
+        || CheckoutPackagePattern.IsMatch(path);
+
     private static string NormalizePath(string path)
     {
         if (string.IsNullOrWhiteSpace(path))
@@ -146,6 +191,9 @@ public class SeoMetadataProvider
 
     private static bool IsNoIndexPath(string path)
     {
+        if (NoIndexPaths.Contains(path))
+            return true;
+
         foreach (var prefix in NoIndexPrefixes)
         {
             if (path.Equals(prefix, StringComparison.OrdinalIgnoreCase)

src/FrontOffice.Main/Utilities/Seo/SitemapGenerator.cs

@@ -91,6 +91,12 @@ public class SitemapGenerator
         return Encoding.UTF8.GetString(stream.ToArray());
     }
 
+    public async Task<byte[]> GenerateBytesAsync(CancellationToken cancellationToken = default)
+    {
+        var xml = await GenerateAsync(cancellationToken);
+        return Encoding.UTF8.GetBytes(xml);
+    }
+
     private static XElement CreateUrlElement(
         XNamespace ns,
         string loc,

src/FrontOffice.Main/wwwroot/robots.txt

@@ -13,5 +13,9 @@ Disallow: /discount-store/orders
 Disallow: /discount-store/order/
 Disallow: /my-orders
 Disallow: /my-cart
+Disallow: /wishlist
+Disallow: /commission/
+Disallow: /network/
+Disallow: /my-packages
 
 Sitemap: https://kbs2.ir/sitemap.xml