Commit 165924b7 authored by Arunoda Susiripala's avatar Arunoda Susiripala Committed by Naoyuki Kanezawa

Add Gzip support for JSON pages (#571)

* Server JSON pages directly from the filesystem.

* Make Json pages even if there's an error.

* Implement much better page serving.

* Use JsonPagesPlugin in the production mode as well.

* Add gzip support for JSON pages.

* Use glob-promise instead of recursive-readdir

* Handle renderStatic 404 properly.

* Simply the gzip code.

* Cache already read JSON pages.

* Change JSON pages extension to .json.

* Fix HMR related issue.

* Fix hot-reload for .json solely on server.

* Properly clear cache on hot-reloader.

* Convert .js pages into .json page right inside the plugin.

* Fix gzipping .json pages.

* Remove unwanted json pages cleanup.

* Get rid of deprecated fs.exists for fs.access
parent 56ca84ba
import fs from 'fs'
import path from 'path'
import zlib from 'zlib'
import glob from 'glob-promise'
export default async function gzipAssets (dir) {
const nextDir = path.resolve(dir, '.next')
await Promise.all([
gzip(path.resolve(nextDir, 'commons.js')),
gzip(path.resolve(nextDir, 'main.js'))
])
const coreAssets = [
path.join(nextDir, 'commons.js'),
path.join(nextDir, 'main.js')
]
const pages = await glob('bundles/pages/**/*.json', { cwd: nextDir })
const allAssets = [
...coreAssets,
...pages.map(page => path.join(nextDir, page))
]
while (true) {
// gzip only 10 assets in parallel at a time.
const currentChunk = allAssets.splice(0, 10)
if (currentChunk.length === 0) break
await Promise.all(currentChunk.map(gzip))
}
}
export function gzip (filePath) {
......
export default class JsonPagesPlugin {
apply (compiler) {
compiler.plugin('after-compile', (compilation, callback) => {
const pages = Object
.keys(compilation.assets)
.filter((filename) => /^bundles\/pages.*\.js$/.test(filename))
pages.forEach((pageName) => {
const page = compilation.assets[pageName]
delete compilation.assets[pageName]
const content = page.source()
const newContent = JSON.stringify({ component: content })
compilation.assets[`${pageName}on`] = {
source: () => newContent,
size: () => newContent.length
}
})
callback()
})
}
}
......@@ -10,6 +10,7 @@ import WatchPagesPlugin from './plugins/watch-pages-plugin'
import WatchRemoveEventPlugin from './plugins/watch-remove-event-plugin'
import DynamicEntryPlugin from './plugins/dynamic-entry-plugin'
import DetachPlugin from './plugins/detach-plugin'
import JsonPagesPlugin from './plugins/json-pages-plugin'
import getConfig from '../config'
const documentPage = join('pages', '_document.js')
......@@ -69,7 +70,8 @@ export default async function createCompiler (dir, { dev = false, quiet = false
name: 'commons',
filename: 'commons.js',
minChunks: Math.max(2, minChunks)
})
}),
new JsonPagesPlugin()
]
if (dev) {
......
......@@ -4,7 +4,7 @@ import webpackHotMiddleware from 'webpack-hot-middleware'
import isWindowsBash from 'is-windows-bash'
import webpack from './build/webpack'
import clean from './build/clean'
import read from './read'
import readPage from './read-page'
export default class HotReloader {
constructor (dir, { quiet } = {}) {
......@@ -182,7 +182,7 @@ export default class HotReloader {
function deleteCache (path) {
delete require.cache[path]
delete read.cache[path]
delete readPage.cache[path]
}
function diff (a, b) {
......
import { resolve, join } from 'path'
import { parse } from 'url'
import http from 'http'
import fs from 'mz/fs'
import send from 'send'
import accepts from 'accepts'
import {
renderToHTML,
renderErrorToHTML,
renderJSON,
renderErrorJSON,
sendHTML
sendHTML,
serveStatic,
serveStaticWithGzip
} from './render'
import Router from './router'
import HotReloader from './hot-reloader'
......@@ -58,32 +57,32 @@ export default class Server {
defineRoutes () {
this.router.get('/_next-prefetcher.js', async (req, res, params) => {
const p = join(__dirname, '../client/next-prefetcher-bundle.js')
await this.serveStatic(req, res, p)
await serveStatic(req, res, p)
})
this.router.get('/_next/main.js', async (req, res, params) => {
const p = join(this.dir, '.next/main.js')
await this.serveStaticWithGzip(req, res, p)
await serveStaticWithGzip(req, res, p)
})
this.router.get('/_next/commons.js', async (req, res, params) => {
const p = join(this.dir, '.next/commons.js')
await this.serveStaticWithGzip(req, res, p)
await serveStaticWithGzip(req, res, p)
})
this.router.get('/_next/pages/:path*', async (req, res, params) => {
const paths = params.path || []
const paths = params.path || ['index']
const pathname = `/${paths.join('/')}`
await this.renderJSON(res, pathname)
await this.renderJSON(req, res, pathname)
})
this.router.get('/_next/:path+', async (req, res, params) => {
const p = join(__dirname, '..', 'client', ...(params.path || []))
await this.serveStatic(req, res, p)
await serveStatic(req, res, p)
})
this.router.get('/static/:path+', async (req, res, params) => {
const p = join(this.dir, 'static', ...(params.path || []))
await this.serveStatic(req, res, p)
await serveStatic(req, res, p)
})
this.router.get('/:path*', async (req, res) => {
......@@ -180,69 +179,38 @@ export default class Server {
this.renderErrorToHTML(null, req, res, pathname, query)
}
async renderJSON (res, page) {
async renderJSON (req, res, page) {
if (this.dev) {
const compilationErr = this.getCompilationError(page)
if (compilationErr) {
return this.renderErrorJSON(compilationErr, res)
return this.renderErrorJSON(compilationErr, req, res)
}
}
try {
await renderJSON(res, page, this.renderOpts)
await renderJSON(req, res, page, this.renderOpts)
} catch (err) {
if (err.code === 'ENOENT') {
res.statusCode = 404
return this.renderErrorJSON(null, res)
return this.renderErrorJSON(null, req, res)
} else {
if (!this.quiet) console.error(err)
res.statusCode = 500
return this.renderErrorJSON(err, res)
return this.renderErrorJSON(err, req, res)
}
}
}
async renderErrorJSON (err, res) {
async renderErrorJSON (err, req, res) {
if (this.dev) {
const compilationErr = this.getCompilationError('/_error')
if (compilationErr) {
res.statusCode = 500
return renderErrorJSON(compilationErr, res, this.renderOpts)
return renderErrorJSON(compilationErr, req, res, this.renderOpts)
}
}
return renderErrorJSON(err, res, this.renderOpts)
}
async serveStaticWithGzip (req, res, path) {
const encoding = accepts(req).encodings(['gzip'])
if (encoding !== 'gzip') {
return this.serveStatic(req, res, path)
}
const gzipPath = `${path}.gz`
const exists = await fs.exists(gzipPath)
if (!exists) {
return this.serveStatic(req, res, path)
}
res.setHeader('Content-Encoding', 'gzip')
return this.serveStatic(req, res, gzipPath)
}
serveStatic (req, res, path) {
return new Promise((resolve, reject) => {
send(req, path)
.on('error', (err) => {
if (err.code === 'ENOENT') {
this.render404(req, res).then(resolve, reject)
} else {
reject(err)
}
})
.pipe(res)
.on('finish', resolve)
})
return renderErrorJSON(err, req, res, this.renderOpts)
}
getCompilationError (page) {
......
......@@ -2,22 +2,24 @@ import fs from 'mz/fs'
import resolve from './resolve'
/**
* resolve a file like `require.resolve`,
* resolve a JSON page like `require.resolve`,
* and read and cache the file content
*/
async function read (path) {
async function readPage (path) {
const f = await resolve(path)
if (cache.hasOwnProperty(f)) {
return cache[f]
}
const data = fs.readFile(f, 'utf8')
cache[f] = data
return data
const source = await fs.readFile(f, 'utf8')
const { component } = JSON.parse(source)
cache[f] = component
return component
}
export default read
export default readPage
export const cache = {}
read.cache = cache
readPage.cache = cache
import { join } from 'path'
import { createElement } from 'react'
import { renderToString, renderToStaticMarkup } from 'react-dom/server'
import fs from 'mz/fs'
import send from 'send'
import accepts from 'accepts'
import requireModule from './require'
import read from './read'
import readPage from './read-page'
import { Router } from '../lib/router'
import Head, { defaultHead } from '../lib/head'
import App from '../lib/app'
......@@ -48,8 +51,8 @@ async function doRender (req, res, pathname, query, {
errorComponent
] = await Promise.all([
Component.getInitialProps ? Component.getInitialProps(ctx) : {},
read(join(dir, '.next', 'bundles', 'pages', page)),
read(join(dir, '.next', 'bundles', 'pages', dev ? '_error-debug' : '_error'))
readPage(join(dir, '.next', 'bundles', 'pages', page)),
readPage(join(dir, '.next', 'bundles', 'pages', dev ? '_error-debug' : '_error'))
])
// the response might be finshed on the getinitialprops call
......@@ -93,14 +96,15 @@ async function doRender (req, res, pathname, query, {
return '<!DOCTYPE html>' + renderToStaticMarkup(doc)
}
export async function renderJSON (res, page, { dir = process.cwd() } = {}) {
const component = await read(join(dir, '.next', 'bundles', 'pages', page))
sendJSON(res, { component })
export async function renderJSON (req, res, page, { dir = process.cwd() } = {}) {
const pagePath = join(dir, '.next', 'bundles', 'pages', `${page}.json`)
return serveStaticWithGzip(req, res, pagePath)
}
export async function renderErrorJSON (err, res, { dir = process.cwd(), dev = false } = {}) {
export async function renderErrorJSON (err, req, res, { dir = process.cwd(), dev = false } = {}) {
const page = err && dev ? '/_error-debug' : '/_error'
const component = await read(join(dir, '.next', 'bundles', 'pages', page))
const component = await readPage(join(dir, '.next', 'bundles', 'pages', page))
sendJSON(res, {
component,
err: err && dev ? errorToJSON(err) : null
......@@ -136,3 +140,44 @@ function errorToJSON (err) {
return json
}
export async function serveStaticWithGzip (req, res, path) {
const encoding = accepts(req).encodings(['gzip'])
if (encoding !== 'gzip') {
return serveStatic(req, res, path)
}
try {
const gzipPath = `${path}.gz`
// fs.access before a file read is not recommeded due to race conditions.
// But in this case, this is totally fine because we know
// it's impossible to have a race condition like that.
// (Since we gzip AOT when called `next build`)
await fs.access(gzipPath, fs.constants.R_OK)
res.setHeader('Content-Encoding', 'gzip')
return serveStatic(req, res, gzipPath)
} catch (ex) {
if (ex.code === 'ENOENT') {
return serveStatic(req, res, path)
}
throw ex
}
}
export function serveStatic (req, res, path) {
return new Promise((resolve, reject) => {
send(req, path)
.on('error', (err) => {
if (err.code === 'ENOENT') {
res.statusCode = 404
res.end('Not Found')
resolve()
} else {
reject(err)
}
})
.pipe(res)
.on('finish', resolve)
})
}
......@@ -26,11 +26,20 @@ function getPaths (id) {
const i = sep === '/' ? id : id.replace(/\//g, sep)
if (i.slice(-3) === '.js') return [i]
if (i[i.length - 1] === sep) return [i + 'index.js']
if (i.slice(-5) === '.json') return [i]
if (i[i.length - 1] === sep) {
return [
i + 'index.json',
i + 'index.js'
]
}
return [
i + '.js',
join(i, 'index.js')
join(i, 'index.js'),
i + '.json',
join(i, 'index.json')
]
}
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment