Commit 165924b7 authored by Arunoda Susiripala's avatar Arunoda Susiripala Committed by Naoyuki Kanezawa

Add Gzip support for JSON pages (#571)

* Server JSON pages directly from the filesystem.

* Make Json pages even if there's an error.

* Implement much better page serving.

* Use JsonPagesPlugin in the production mode as well.

* Add gzip support for JSON pages.

* Use glob-promise instead of recursive-readdir

* Handle renderStatic 404 properly.

* Simply the gzip code.

* Cache already read JSON pages.

* Change JSON pages extension to .json.

* Fix HMR related issue.

* Fix hot-reload for .json solely on server.

* Properly clear cache on hot-reloader.

* Convert .js pages into .json page right inside the plugin.

* Fix gzipping .json pages.

* Remove unwanted json pages cleanup.

* Get rid of deprecated fs.exists for fs.access
parent 56ca84ba
import fs from 'fs' import fs from 'fs'
import path from 'path' import path from 'path'
import zlib from 'zlib' import zlib from 'zlib'
import glob from 'glob-promise'
export default async function gzipAssets (dir) { export default async function gzipAssets (dir) {
const nextDir = path.resolve(dir, '.next') const nextDir = path.resolve(dir, '.next')
await Promise.all([ const coreAssets = [
gzip(path.resolve(nextDir, 'commons.js')), path.join(nextDir, 'commons.js'),
gzip(path.resolve(nextDir, 'main.js')) path.join(nextDir, 'main.js')
]) ]
const pages = await glob('bundles/pages/**/*.json', { cwd: nextDir })
const allAssets = [
...coreAssets,
...pages.map(page => path.join(nextDir, page))
]
while (true) {
// gzip only 10 assets in parallel at a time.
const currentChunk = allAssets.splice(0, 10)
if (currentChunk.length === 0) break
await Promise.all(currentChunk.map(gzip))
}
} }
export function gzip (filePath) { export function gzip (filePath) {
......
export default class JsonPagesPlugin {
apply (compiler) {
compiler.plugin('after-compile', (compilation, callback) => {
const pages = Object
.keys(compilation.assets)
.filter((filename) => /^bundles\/pages.*\.js$/.test(filename))
pages.forEach((pageName) => {
const page = compilation.assets[pageName]
delete compilation.assets[pageName]
const content = page.source()
const newContent = JSON.stringify({ component: content })
compilation.assets[`${pageName}on`] = {
source: () => newContent,
size: () => newContent.length
}
})
callback()
})
}
}
...@@ -10,6 +10,7 @@ import WatchPagesPlugin from './plugins/watch-pages-plugin' ...@@ -10,6 +10,7 @@ import WatchPagesPlugin from './plugins/watch-pages-plugin'
import WatchRemoveEventPlugin from './plugins/watch-remove-event-plugin' import WatchRemoveEventPlugin from './plugins/watch-remove-event-plugin'
import DynamicEntryPlugin from './plugins/dynamic-entry-plugin' import DynamicEntryPlugin from './plugins/dynamic-entry-plugin'
import DetachPlugin from './plugins/detach-plugin' import DetachPlugin from './plugins/detach-plugin'
import JsonPagesPlugin from './plugins/json-pages-plugin'
import getConfig from '../config' import getConfig from '../config'
const documentPage = join('pages', '_document.js') const documentPage = join('pages', '_document.js')
...@@ -69,7 +70,8 @@ export default async function createCompiler (dir, { dev = false, quiet = false ...@@ -69,7 +70,8 @@ export default async function createCompiler (dir, { dev = false, quiet = false
name: 'commons', name: 'commons',
filename: 'commons.js', filename: 'commons.js',
minChunks: Math.max(2, minChunks) minChunks: Math.max(2, minChunks)
}) }),
new JsonPagesPlugin()
] ]
if (dev) { if (dev) {
......
...@@ -4,7 +4,7 @@ import webpackHotMiddleware from 'webpack-hot-middleware' ...@@ -4,7 +4,7 @@ import webpackHotMiddleware from 'webpack-hot-middleware'
import isWindowsBash from 'is-windows-bash' import isWindowsBash from 'is-windows-bash'
import webpack from './build/webpack' import webpack from './build/webpack'
import clean from './build/clean' import clean from './build/clean'
import read from './read' import readPage from './read-page'
export default class HotReloader { export default class HotReloader {
constructor (dir, { quiet } = {}) { constructor (dir, { quiet } = {}) {
...@@ -182,7 +182,7 @@ export default class HotReloader { ...@@ -182,7 +182,7 @@ export default class HotReloader {
function deleteCache (path) { function deleteCache (path) {
delete require.cache[path] delete require.cache[path]
delete read.cache[path] delete readPage.cache[path]
} }
function diff (a, b) { function diff (a, b) {
......
import { resolve, join } from 'path' import { resolve, join } from 'path'
import { parse } from 'url' import { parse } from 'url'
import http from 'http' import http from 'http'
import fs from 'mz/fs'
import send from 'send'
import accepts from 'accepts'
import { import {
renderToHTML, renderToHTML,
renderErrorToHTML, renderErrorToHTML,
renderJSON, renderJSON,
renderErrorJSON, renderErrorJSON,
sendHTML sendHTML,
serveStatic,
serveStaticWithGzip
} from './render' } from './render'
import Router from './router' import Router from './router'
import HotReloader from './hot-reloader' import HotReloader from './hot-reloader'
...@@ -58,32 +57,32 @@ export default class Server { ...@@ -58,32 +57,32 @@ export default class Server {
defineRoutes () { defineRoutes () {
this.router.get('/_next-prefetcher.js', async (req, res, params) => { this.router.get('/_next-prefetcher.js', async (req, res, params) => {
const p = join(__dirname, '../client/next-prefetcher-bundle.js') const p = join(__dirname, '../client/next-prefetcher-bundle.js')
await this.serveStatic(req, res, p) await serveStatic(req, res, p)
}) })
this.router.get('/_next/main.js', async (req, res, params) => { this.router.get('/_next/main.js', async (req, res, params) => {
const p = join(this.dir, '.next/main.js') const p = join(this.dir, '.next/main.js')
await this.serveStaticWithGzip(req, res, p) await serveStaticWithGzip(req, res, p)
}) })
this.router.get('/_next/commons.js', async (req, res, params) => { this.router.get('/_next/commons.js', async (req, res, params) => {
const p = join(this.dir, '.next/commons.js') const p = join(this.dir, '.next/commons.js')
await this.serveStaticWithGzip(req, res, p) await serveStaticWithGzip(req, res, p)
}) })
this.router.get('/_next/pages/:path*', async (req, res, params) => { this.router.get('/_next/pages/:path*', async (req, res, params) => {
const paths = params.path || [] const paths = params.path || ['index']
const pathname = `/${paths.join('/')}` const pathname = `/${paths.join('/')}`
await this.renderJSON(res, pathname) await this.renderJSON(req, res, pathname)
}) })
this.router.get('/_next/:path+', async (req, res, params) => { this.router.get('/_next/:path+', async (req, res, params) => {
const p = join(__dirname, '..', 'client', ...(params.path || [])) const p = join(__dirname, '..', 'client', ...(params.path || []))
await this.serveStatic(req, res, p) await serveStatic(req, res, p)
}) })
this.router.get('/static/:path+', async (req, res, params) => { this.router.get('/static/:path+', async (req, res, params) => {
const p = join(this.dir, 'static', ...(params.path || [])) const p = join(this.dir, 'static', ...(params.path || []))
await this.serveStatic(req, res, p) await serveStatic(req, res, p)
}) })
this.router.get('/:path*', async (req, res) => { this.router.get('/:path*', async (req, res) => {
...@@ -180,69 +179,38 @@ export default class Server { ...@@ -180,69 +179,38 @@ export default class Server {
this.renderErrorToHTML(null, req, res, pathname, query) this.renderErrorToHTML(null, req, res, pathname, query)
} }
async renderJSON (res, page) { async renderJSON (req, res, page) {
if (this.dev) { if (this.dev) {
const compilationErr = this.getCompilationError(page) const compilationErr = this.getCompilationError(page)
if (compilationErr) { if (compilationErr) {
return this.renderErrorJSON(compilationErr, res) return this.renderErrorJSON(compilationErr, req, res)
} }
} }
try { try {
await renderJSON(res, page, this.renderOpts) await renderJSON(req, res, page, this.renderOpts)
} catch (err) { } catch (err) {
if (err.code === 'ENOENT') { if (err.code === 'ENOENT') {
res.statusCode = 404 res.statusCode = 404
return this.renderErrorJSON(null, res) return this.renderErrorJSON(null, req, res)
} else { } else {
if (!this.quiet) console.error(err) if (!this.quiet) console.error(err)
res.statusCode = 500 res.statusCode = 500
return this.renderErrorJSON(err, res) return this.renderErrorJSON(err, req, res)
} }
} }
} }
async renderErrorJSON (err, res) { async renderErrorJSON (err, req, res) {
if (this.dev) { if (this.dev) {
const compilationErr = this.getCompilationError('/_error') const compilationErr = this.getCompilationError('/_error')
if (compilationErr) { if (compilationErr) {
res.statusCode = 500 res.statusCode = 500
return renderErrorJSON(compilationErr, res, this.renderOpts) return renderErrorJSON(compilationErr, req, res, this.renderOpts)
} }
} }
return renderErrorJSON(err, res, this.renderOpts) return renderErrorJSON(err, req, res, this.renderOpts)
}
async serveStaticWithGzip (req, res, path) {
const encoding = accepts(req).encodings(['gzip'])
if (encoding !== 'gzip') {
return this.serveStatic(req, res, path)
}
const gzipPath = `${path}.gz`
const exists = await fs.exists(gzipPath)
if (!exists) {
return this.serveStatic(req, res, path)
}
res.setHeader('Content-Encoding', 'gzip')
return this.serveStatic(req, res, gzipPath)
}
serveStatic (req, res, path) {
return new Promise((resolve, reject) => {
send(req, path)
.on('error', (err) => {
if (err.code === 'ENOENT') {
this.render404(req, res).then(resolve, reject)
} else {
reject(err)
}
})
.pipe(res)
.on('finish', resolve)
})
} }
getCompilationError (page) { getCompilationError (page) {
......
...@@ -2,22 +2,24 @@ import fs from 'mz/fs' ...@@ -2,22 +2,24 @@ import fs from 'mz/fs'
import resolve from './resolve' import resolve from './resolve'
/** /**
* resolve a file like `require.resolve`, * resolve a JSON page like `require.resolve`,
* and read and cache the file content * and read and cache the file content
*/ */
async function read (path) { async function readPage (path) {
const f = await resolve(path) const f = await resolve(path)
if (cache.hasOwnProperty(f)) { if (cache.hasOwnProperty(f)) {
return cache[f] return cache[f]
} }
const data = fs.readFile(f, 'utf8') const source = await fs.readFile(f, 'utf8')
cache[f] = data const { component } = JSON.parse(source)
return data
cache[f] = component
return component
} }
export default read export default readPage
export const cache = {} export const cache = {}
read.cache = cache readPage.cache = cache
import { join } from 'path' import { join } from 'path'
import { createElement } from 'react' import { createElement } from 'react'
import { renderToString, renderToStaticMarkup } from 'react-dom/server' import { renderToString, renderToStaticMarkup } from 'react-dom/server'
import fs from 'mz/fs'
import send from 'send'
import accepts from 'accepts'
import requireModule from './require' import requireModule from './require'
import read from './read' import readPage from './read-page'
import { Router } from '../lib/router' import { Router } from '../lib/router'
import Head, { defaultHead } from '../lib/head' import Head, { defaultHead } from '../lib/head'
import App from '../lib/app' import App from '../lib/app'
...@@ -48,8 +51,8 @@ async function doRender (req, res, pathname, query, { ...@@ -48,8 +51,8 @@ async function doRender (req, res, pathname, query, {
errorComponent errorComponent
] = await Promise.all([ ] = await Promise.all([
Component.getInitialProps ? Component.getInitialProps(ctx) : {}, Component.getInitialProps ? Component.getInitialProps(ctx) : {},
read(join(dir, '.next', 'bundles', 'pages', page)), readPage(join(dir, '.next', 'bundles', 'pages', page)),
read(join(dir, '.next', 'bundles', 'pages', dev ? '_error-debug' : '_error')) readPage(join(dir, '.next', 'bundles', 'pages', dev ? '_error-debug' : '_error'))
]) ])
// the response might be finshed on the getinitialprops call // the response might be finshed on the getinitialprops call
...@@ -93,14 +96,15 @@ async function doRender (req, res, pathname, query, { ...@@ -93,14 +96,15 @@ async function doRender (req, res, pathname, query, {
return '<!DOCTYPE html>' + renderToStaticMarkup(doc) return '<!DOCTYPE html>' + renderToStaticMarkup(doc)
} }
export async function renderJSON (res, page, { dir = process.cwd() } = {}) { export async function renderJSON (req, res, page, { dir = process.cwd() } = {}) {
const component = await read(join(dir, '.next', 'bundles', 'pages', page)) const pagePath = join(dir, '.next', 'bundles', 'pages', `${page}.json`)
sendJSON(res, { component }) return serveStaticWithGzip(req, res, pagePath)
} }
export async function renderErrorJSON (err, res, { dir = process.cwd(), dev = false } = {}) { export async function renderErrorJSON (err, req, res, { dir = process.cwd(), dev = false } = {}) {
const page = err && dev ? '/_error-debug' : '/_error' const page = err && dev ? '/_error-debug' : '/_error'
const component = await read(join(dir, '.next', 'bundles', 'pages', page)) const component = await readPage(join(dir, '.next', 'bundles', 'pages', page))
sendJSON(res, { sendJSON(res, {
component, component,
err: err && dev ? errorToJSON(err) : null err: err && dev ? errorToJSON(err) : null
...@@ -136,3 +140,44 @@ function errorToJSON (err) { ...@@ -136,3 +140,44 @@ function errorToJSON (err) {
return json return json
} }
export async function serveStaticWithGzip (req, res, path) {
const encoding = accepts(req).encodings(['gzip'])
if (encoding !== 'gzip') {
return serveStatic(req, res, path)
}
try {
const gzipPath = `${path}.gz`
// fs.access before a file read is not recommeded due to race conditions.
// But in this case, this is totally fine because we know
// it's impossible to have a race condition like that.
// (Since we gzip AOT when called `next build`)
await fs.access(gzipPath, fs.constants.R_OK)
res.setHeader('Content-Encoding', 'gzip')
return serveStatic(req, res, gzipPath)
} catch (ex) {
if (ex.code === 'ENOENT') {
return serveStatic(req, res, path)
}
throw ex
}
}
export function serveStatic (req, res, path) {
return new Promise((resolve, reject) => {
send(req, path)
.on('error', (err) => {
if (err.code === 'ENOENT') {
res.statusCode = 404
res.end('Not Found')
resolve()
} else {
reject(err)
}
})
.pipe(res)
.on('finish', resolve)
})
}
...@@ -26,11 +26,20 @@ function getPaths (id) { ...@@ -26,11 +26,20 @@ function getPaths (id) {
const i = sep === '/' ? id : id.replace(/\//g, sep) const i = sep === '/' ? id : id.replace(/\//g, sep)
if (i.slice(-3) === '.js') return [i] if (i.slice(-3) === '.js') return [i]
if (i[i.length - 1] === sep) return [i + 'index.js'] if (i.slice(-5) === '.json') return [i]
if (i[i.length - 1] === sep) {
return [
i + 'index.json',
i + 'index.js'
]
}
return [ return [
i + '.js', i + '.js',
join(i, 'index.js') join(i, 'index.js'),
i + '.json',
join(i, 'index.json')
] ]
} }
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment