Unverified Commit 7f57d4b1 authored by Zack Tanner's avatar Zack Tanner Committed by GitHub

send initialCanonicalUrl in array format to fix crawler issues (#69370)

The initial RSC payload currently provides the `initialCanonicalUrl`
value to initialize the router state during the SSR render. However as
noted in #53274, it seems crawlers like Googlebot will crawl anything
URL-like that it finds in the document, regardless of whether or not
it's actually rendered as a link somewhere (which this isn't).

This splits the `initialCanonicalUrl` value into parts and then joins it
when it's ready to be consumed. That way when it's embedded in the
document, it's not in a path-like format.


Fixes #53274
Closes NDX-243
parent bca90db6
...@@ -171,7 +171,7 @@ const pendingActionQueue: Promise<AppRouterActionQueue> = new Promise( ...@@ -171,7 +171,7 @@ const pendingActionQueue: Promise<AppRouterActionQueue> = new Promise(
createInitialRouterState({ createInitialRouterState({
buildId: initialRSCPayload.b, buildId: initialRSCPayload.b,
initialFlightData: initialRSCPayload.f, initialFlightData: initialRSCPayload.f,
initialCanonicalUrl: initialRSCPayload.c, initialCanonicalUrlParts: initialRSCPayload.c,
initialParallelRoutes: new Map(), initialParallelRoutes: new Map(),
location: window.location, location: window.location,
couldBeIntercepted: initialRSCPayload.i, couldBeIntercepted: initialRSCPayload.i,
......
...@@ -38,7 +38,7 @@ describe('createInitialRouterState', () => { ...@@ -38,7 +38,7 @@ describe('createInitialRouterState', () => {
initialFlightData: [ initialFlightData: [
[initialTree, ['', children, {}, null], <title>Test</title>], [initialTree, ['', children, {}, null], <title>Test</title>],
], ],
initialCanonicalUrl, initialCanonicalUrlParts: initialCanonicalUrl.split('/'),
initialParallelRoutes, initialParallelRoutes,
location: new URL('/linking', 'https://localhost') as any, location: new URL('/linking', 'https://localhost') as any,
couldBeIntercepted: false, couldBeIntercepted: false,
...@@ -50,7 +50,7 @@ describe('createInitialRouterState', () => { ...@@ -50,7 +50,7 @@ describe('createInitialRouterState', () => {
initialFlightData: [ initialFlightData: [
[initialTree, ['', children, {}, null], <title>Test</title>], [initialTree, ['', children, {}, null], <title>Test</title>],
], ],
initialCanonicalUrl, initialCanonicalUrlParts: initialCanonicalUrl.split('/'),
initialParallelRoutes, initialParallelRoutes,
location: new URL('/linking', 'https://localhost') as any, location: new URL('/linking', 'https://localhost') as any,
couldBeIntercepted: false, couldBeIntercepted: false,
......
...@@ -10,7 +10,7 @@ import { addRefreshMarkerToActiveParallelSegments } from './refetch-inactive-par ...@@ -10,7 +10,7 @@ import { addRefreshMarkerToActiveParallelSegments } from './refetch-inactive-par
export interface InitialRouterStateParameters { export interface InitialRouterStateParameters {
buildId: string buildId: string
initialCanonicalUrl: string initialCanonicalUrlParts: string[]
initialParallelRoutes: CacheNode['parallelRoutes'] initialParallelRoutes: CacheNode['parallelRoutes']
initialFlightData: FlightDataPath[] initialFlightData: FlightDataPath[]
location: Location | null location: Location | null
...@@ -21,12 +21,16 @@ export interface InitialRouterStateParameters { ...@@ -21,12 +21,16 @@ export interface InitialRouterStateParameters {
export function createInitialRouterState({ export function createInitialRouterState({
buildId, buildId,
initialFlightData, initialFlightData,
initialCanonicalUrl, initialCanonicalUrlParts,
initialParallelRoutes, initialParallelRoutes,
location, location,
couldBeIntercepted, couldBeIntercepted,
postponed, postponed,
}: InitialRouterStateParameters) { }: InitialRouterStateParameters) {
// When initialized on the server, the canonical URL is provided as an array of parts.
// This is to ensure that when the RSC payload streamed to the client, crawlers don't interpret it
// as a URL that should be crawled.
const initialCanonicalUrl = initialCanonicalUrlParts.join('/')
// The initialFlightData is an array of FlightDataPath arrays. // The initialFlightData is an array of FlightDataPath arrays.
// For the root render, there'll only be a top-level FlightDataPath array. // For the root render, there'll only be a top-level FlightDataPath array.
const [initialTree, initialSeedData, initialHead] = initialFlightData[0] const [initialTree, initialSeedData, initialHead] = initialFlightData[0]
......
...@@ -482,6 +482,16 @@ async function generateDynamicFlightRenderResult( ...@@ -482,6 +482,16 @@ async function generateDynamicFlightRenderResult(
}) })
} }
/**
* Crawlers will inadvertently think the canonicalUrl in the RSC payload should be crawled
* when our intention is to just seed the router state with the current URL.
* This function splits up the pathname so that we can later join it on
* when we're ready to consume the path.
*/
function prepareInitialCanonicalUrl(url: RequestStore['url']) {
return (url.pathname + url.search).split('/')
}
// This is the data necessary to render <AppRouter /> when no SSR errors are encountered // This is the data necessary to render <AppRouter /> when no SSR errors are encountered
async function getRSCPayload( async function getRSCPayload(
tree: LoaderTree, tree: LoaderTree,
...@@ -558,7 +568,7 @@ async function getRSCPayload( ...@@ -558,7 +568,7 @@ async function getRSCPayload(
P: <Preloads preloadCallbacks={preloadCallbacks} />, P: <Preloads preloadCallbacks={preloadCallbacks} />,
b: ctx.renderOpts.buildId, b: ctx.renderOpts.buildId,
p: ctx.assetPrefix, p: ctx.assetPrefix,
c: url.pathname + url.search, c: prepareInitialCanonicalUrl(url),
i: !!couldBeIntercepted, i: !!couldBeIntercepted,
f: [[initialTree, seedData, initialHead]], f: [[initialTree, seedData, initialHead]],
m: missingSlots, m: missingSlots,
...@@ -635,7 +645,7 @@ async function getErrorRSCPayload( ...@@ -635,7 +645,7 @@ async function getErrorRSCPayload(
return { return {
b: ctx.renderOpts.buildId, b: ctx.renderOpts.buildId,
p: ctx.assetPrefix, p: ctx.assetPrefix,
c: url.pathname + url.search, c: prepareInitialCanonicalUrl(url),
m: undefined, m: undefined,
i: false, i: false,
f: [[initialTree, initialSeedData, initialHead]], f: [[initialTree, initialSeedData, initialHead]],
...@@ -669,7 +679,7 @@ function App<T>({ ...@@ -669,7 +679,7 @@ function App<T>({
const initialState = createInitialRouterState({ const initialState = createInitialRouterState({
buildId: response.b, buildId: response.b,
initialFlightData: response.f, initialFlightData: response.f,
initialCanonicalUrl: response.c, initialCanonicalUrlParts: response.c,
// location and initialParallelRoutes are not initialized in the SSR render // location and initialParallelRoutes are not initialized in the SSR render
// they are set to an empty map and window.location, respectively during hydration // they are set to an empty map and window.location, respectively during hydration
initialParallelRoutes: null!, initialParallelRoutes: null!,
...@@ -727,7 +737,7 @@ function AppWithoutContext<T>({ ...@@ -727,7 +737,7 @@ function AppWithoutContext<T>({
const initialState = createInitialRouterState({ const initialState = createInitialRouterState({
buildId: response.b, buildId: response.b,
initialFlightData: response.f, initialFlightData: response.f,
initialCanonicalUrl: response.c, initialCanonicalUrlParts: response.c,
// location and initialParallelRoutes are not initialized in the SSR render // location and initialParallelRoutes are not initialized in the SSR render
// they are set to an empty map and window.location, respectively during hydration // they are set to an empty map and window.location, respectively during hydration
initialParallelRoutes: null!, initialParallelRoutes: null!,
......
...@@ -206,8 +206,8 @@ export type InitialRSCPayload = { ...@@ -206,8 +206,8 @@ export type InitialRSCPayload = {
b: string b: string
/** assetPrefix */ /** assetPrefix */
p: string p: string
/** initialCanonicalUrl */ /** initialCanonicalUrlParts */
c: string c: string[]
/** couldBeIntercepted */ /** couldBeIntercepted */
i: boolean i: boolean
/** initialFlightData */ /** initialFlightData */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment