robots.txt is not valid

[RemyJouni]

Description

Hi, I'm using next-intl 3.0.0-rc.2, and it's been working great for me. However, when I tested my website's speed on PageSpeed Insight, my SEO ranking decreased due to an error message that said "robots.txt is not valid". This issue only occurred after I merged my i18n changes.

The CodeSandbox code is nearly identical in terms of structure to my website.

I suspect that it might be related to my middleware code.

Mandatory reproduction URL (CodeSandbox or GitHub repository)

https://codesandbox.io/p/sandbox/next-intl-bug-template-app-forked-qmtnyl?file=%2Fsrc%2Fmiddleware.ts%3A8%2C1

Reproduction description

I tried to run the PageSpeed test on CodeSandbox, but unfortunately, it didn't run.

But you can take a look at my test results before the i18n merge from here.

And here is the test results after merging the i18n.

This is the full error message that was given to me.
`

<script src="/_next/static/chunks/fd9d1056-57c91f8d27b6ce2a.js" async=""></script><script src="/_next/static/chunks/864-5e5a810d40da0268.js" async=""></script><script src="/_next/static/chunks/main-app-8f13c08947745519.js" async=""></script><title>Gulf Picasso</title><script src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" noModule=""></script><script src="/_next/static/chunks/webpack-9d77abe06a5976d6.js" async=""></script><script>(self.__next_f=self.__next_f\|\|[]).push([0])</script><script>self.__next_f.push([1,"0:\"$L1\"\n"])</script><script>self.__next_f.push([1,"2:HL[\"/_next/static/media/01af0fc7b4278e65-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n3:HL[\"/_next/static/media/0cfcae2a88746a80-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n4:HL[\"/_next/static/media/1f22effff9a6344a-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n5:HL[\"/_next/static/media/59bb3598b57bad1e-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n6:HL[\"/_next/static/media/80236724d32a4a00-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n7:HL[\"/"])</script><script>self.__next_f.push([1,"_next/static/media/e728cd9385374b10-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n8:HL[\"/_next/static/css/1d2c10921ef72638.css\",\"style\"]\n9:HL[\"/_next/static/css/bbd903629053cd21.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"a:I{\"id\":6054,\"chunks\":[\"272:static/chunks/webpack-9d77abe06a5976d6.js\",\"971:static/chunks/fd9d1056-57c91f8d27b6ce2a.js\",\"864:static/chunks/864-5e5a810d40da0268.js\"],\"name\":\"\",\"async\":false}\nc:I{\"id\":1729,\"chunks\":[\"272:static/chunks/webpack-9d77abe06a5976d6.js\",\"971:static/chunks/fd9d1056-57c91f8d27b6ce2a.js\",\"864:static/chunks/864-5e5a810d40da0268.js\"],\"name\":\"\",\"async\":false}\nd:I{\"id\":1443,\"chunks\":[\"272:static/chunks/webpack-9d77abe06a5976d6.js\",\"971:static/chunks/fd9d1056-57c91f8d27b6ce2a.js\",\"864:stat"])</script><script>self.__next_f.push([1,"ic/chunks/864-5e5a810d40da0268.js\"],\"name\":\"\",\"async\":false}\ne:I{\"id\":8639,\"chunks\":[\"272:static/chunks/webpack-9d77abe06a5976d6.js\",\"971:static/chunks/fd9d1056-57c91f8d27b6ce2a.js\",\"864:static/chunks/864-5e5a810d40da0268.js\"],\"name\":\"\",\"async\":false}\nf:I{\"id\":4244,\"chunks\":[\"724:static/chunks/724-c7d29b59e420ad88.js\",\"58:static/chunks/58-e45206503a5e3833.js\",\"145:static/chunks/145-d862029237a2ed0a.js\",\"898:static/chunks/898-8918bba375f3671b.js\",\"203:static/chunks/app/[locale]/layout-423c62fac9862230.js\"],\""])</script><script>self.__next_f.push([1,"name\":\"\",\"async\":false}\n10:I{\"id\":8655,\"chunks\":[\"724:static/chunks/724-c7d29b59e420ad88.js\",\"58:static/chunks/58-e45206503a5e3833.js\",\"145:static/chunks/145-d862029237a2ed0a.js\",\"898:static/chunks/898-8918bba375f3671b.js\",\"203:static/chunks/app/[locale]/layout-423c62fac9862230.js\"],\"name\":\"TokensCountProvider\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"1:[[],[\"$\",\"$La\",null,{\"buildId\":\"ql1Wg8p9mwn2lEexEmEGQ\",\"assetPrefix\":\"\",\"initialCanonicalUrl\":\"/robots.txt\",\"initialTree\":[\"\",{\"children\":[[\"locale\",\"robots.txt\",\"d\"],{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true]}],\"initialHead\":[false,\"$Lb\"],\"globalErrorComponent\":\"$c\",\"children\":[\"$\",\"$Ld\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"template\":[\"$\",\"$Le\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"childProp\":{\"current\":[null,[\"$\",\"html\",null,{\"lang\":\"robots.txt\",\"dir\":\"ltr\",\"className\":\"__variable_1b7341 __variable_9da26a __variable_7c6c75\",\"children\":[[[\"$\",\"$Lf\",null,{\"src\":\"https://www.googletagmanager.com/gtag/js?id=G-RPPBPQH47S\",\"strategy\":\"afterInteractive\"}],[\"$\",\"$Lf\",null,{\"id\":\"google-analytics\",\"strategy\":\"afterInteractive\",\"children\":\"\\n window.dataLayer = window.dataLayer \|\| [];\\n function gtag(){dataLayer.push(arguments);}\\n gtag('js', new Date());\\n\\n gtag('config', 'G-RPPBPQH47S');\\n \"}]],[\"$\",\"$Lf\",null,{\"id\":\"hotjar-analytics\",\"strategy\":\"afterInteractive\",\"children\":\"\\n (function(h,o,t,j,a,r){\\n h.hj=h.hj\|\|function(){(h.hj.q=h.hj.q\|\|[]).push(arguments)};\\n h._hjSettings={hjid:3677915,hjsv:6};\\n a=o.getElementsByTagName('head')[0];\\n r=o.createElement('script');r.async=1;\\n r.src=t+h._hjSettings.hjid+j+h._hjSettings.hjsv;\\n a.appendChild(r);\\n })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');\\n \"}],[\"$\",\"body\",null,{\"className\":\" bg-gulf-900 text-gulf-300 min-h-screen w-full flex flex-col items-center text-base font-urbanist\",\"children\":[\"$\",\"$L10\",null,{\"children\":[\"$L11\",[\"$\",\"$Ld\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",[\"locale\",\"robots.txt\",\"d\"],\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"template\":[\"$\",\"$Le\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":\"$L12\",\"notFoundStyles\":[],\"childProp\":{\"current\":[\"$L13\",\"$L14\",null],\"segment\":\"__PAGE__\"},\"styles\":[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/bbd903629053cd21.css\",\"precedence\":\"next\"}]]}],\"$L15\"]}]}]]}],null],\"segment\":[\"locale\",\"robots.txt\",\"d\"]},\"styles\":[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/1d2c10921ef72638.css\",\"precedence\":\"next\"}]]}]}]]\n"])</script><script>self.__next_f.push([1,"11:E{\"digest\":\"3776845038\"}\n12:E{\"digest\":\"3776845038\"}\n14:E{\"digest\":\"3776845038\"}\n15:E{\"digest\":\"3776845038\"}\n"])</script><script>self.__next_f.push([1,"b:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"1\",{\"children\":\"Gulf Picasso\"}],[\"$\",\"meta\",\"2\",{\"name\":\"description\",\"content\":\"Free AI Avatar Creator\"}],[\"$\",\"meta\",\"3\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:image:type\",\"content\":\"image/jpeg\"}],[\"$\",\"meta\",\"5\",{\"property\":\"og:image:width\",\"content\":\"1731\"}],[\"$\",\"meta\",\"6\",{\"property\":\"og:image:height\",\"content\":\"1088\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:image\",\"content\":\"https://gulf-picasso-avatar-1d5oqoevz-engremi.vercel.app/opengraph-image.jpg?c702fab57c01f60a\"}],[\"$\",\"meta\",\"8\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"9\",{\"name\":\"twitter:image:type\",\"content\":\"image/jpeg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:image:width\",\"content\":\"1731\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:image:height\",\"content\":\"1088\"}],[\"$\",\"meta\",\"12\",{\"name\":\"twitter:image\",\"content\":\"https://gulf-picasso-avatar-1d5oqoevz-engremi.vercel.app/twitter-image.jpg?c702fab57c01f60a\"}],[\"$\",\"link\",\"13\",{\"rel\":\"icon\",\"href\":\"/icon.png?29b886bd957bff1a\",\"type\":\"image/png\",\"sizes\":\"192x192\"}],[\"$\",\"link\",\"14\",{\"rel\":\"apple-touch-icon\",\"href\":\"/apple-icon.png?947ff744f3d607a1\",\"type\":\"image/png\",\"sizes\":\"180x180\"}],[\"$\",\"meta\",\"15\",{\"name\":\"next-size-adjust\"}]]\n"])</script><script>self.__next_f.push([1,"13:null\n"])</script>

--

`

Expected behaviour

Not to have a robot.txt error if I'm using the next-intl library.

And thank you very much in advance, you all are amazing for creating such a great library.

[amannn]

Thank you for the report! I've just added a robots.txt to example-next-13 in a256429 to serve as a reference. You might want to ensure you're using the latest version of Next.js, I hope this helps!

[RemyJouni]

Thank you very much, it's perfect now!