Anchor link Installation
Anchor link Requirements
Microservices in the DADI platform are built on Node.js, a JavaScript runtime built on Google Chrome's V8 JavaScript engine. Node.js uses an event-driven, non-blocking I/O model that makes it lightweight and efficient.
DADI follows the Node.js LTS (Long Term Support) release schedule, and as such the version of Node.js required to run DADI products is coupled to the version of Node.js currently in Active LTS. See the LTS schedule for further information.
- Node.js (current LTS version)
Anchor link DADI CLI
The easiest way to install Web is using DADI CLI. CLI is a command line application that can be used to create and maintain installations of DADI products.
Anchor link Install DADI CLI
$ npm install @dadi/cli -g
Anchor link Create new Web installation
There are two ways to create a new Web application with the CLI: either manually create a new directory for Web or let CLI handle that for you. DADI CLI accepts an argument for project-name
which it uses to create a directory for installation.
Manual directory creation
$ mkdir my-web-app
$ cd my-web-app
$ dadi web new
Automatic directory creation
$ dadi web new my-web-app
$ cd my-web-app
Anchor link NPM
All DADI platform microservices are available from NPM. To add Web to your existing project as a dependency:
$ cd my-existing-node-app
$ npm install @dadi/web
This will create config
& workspace
folders and server.js
which will serve as the entry point to your app.
Anchor link Application Anatomy
When CLI finishes creating your Web instance, the application directory will contain the basic requirements for launching your Web instance. The following directories and files have been created for you:
my-web/
config/ # contains environment-specific configuration files
config.development.json
server.js # the entry point for the application
package.json
workspace/
datasources/ # datasource specification files
events/ # event files - files run before page render
pages/ # page template and specification (.json) files
partials/ # page template includes
posts/ # markdown files as a blog example
public/ # files to expose raw to the web, e.g. icons, images, styles, scripts
utils/
helpers/ # globally accessible functions that templates can call
collections/ # collection specification files
endpoints/ # custom JavaScript endpoints
Anchor link Configuration
All the core platform services are configured using environment specific configuration.json files, the default being development
. For more advanced users this can also load based on the hostname
i.e., it will also look for config." + req.headers.host + ".json
The minimal config.development.json
file looks like this:
{
"server": {
"host": "localhost",
"port": 3000
},
"cluster": false
}
Anchor link Advanced configuration
Anchor link Example Configuration File
{
"app": {
"name": "Project Name Here"
},
"server": {
"host": "127.0.0.1",
"port": 443,
"socketTimeoutSec": 30,
"protocol": "https",
"redirectPort": 80,
"sslPassphrase": "superSecretPassphrase",
"sslPrivateKeyPath": "keys/server.key",
"sslCertificatePath": "keys/server.crt"
},
"api": {
"host": "127.0.0.1",
"port": 3000
},
"auth": {
"tokenUrl":"/token",
"clientId":"webClient",
"secret":"secretSquirrel"
},
"aws": {
"accessKeyId": "<your key here>",
"secretAccessKey": "<your secret here>",
"region": "eu-west-1"
},
"caching": {
"ttl": 300,
"directory": {
"enabled": true,
"path": "./cache/web/",
"extension": "html"
},
"redis": {
"enabled": false,
"host": "localhost",
"port": 6379
}
},
"engines": {
"dust": {
"cache": true,
"debug": true,
"debugLevel": "DEBUG",
"whitespace": true,
"paths": {
"helpers": "workspace/utils/helpers"
}
}
},
"headers": {
"useCompression": true,
"cacheControl": {
"text/css": "public, max-age=86400"
}
},
"logging": {
"enabled": true,
"level": "info",
"path": "./log",
"filename": "dadi-web",
"extension": "log",
"accessLog": {
"enabled": true,
"kinesisStream": "dadi_web_test_stream"
}
},
"paths": {
"datasources": "./workspace/datasources",
"events": "./workspace/events",
"middleware": "./workspace/middleware",
"pages": "./workspace/pages",
"partials": "./workspace/partials",
"public": "./workspace/public",
"routes": "./workspace/routes",
"helpers": "./workspace/utils/helpers",
"filters": "./workspace/utils/filters"
},
"rewrites": {
"datasource": "redirects",
"path": "workspace/routes/rewrites.txt",
"forceLowerCase": true,
"forceTrailingSlash": true,
"stripIndexPages": ['index.php', 'default.aspx']
},
"global" : {
"baseUrl": "http://www.example.com"
},
"globalEvents": [
"timestamp"
],
"debug": true,
"allowDebugView": true
}
You can see all the config options in config.js
.
Anchor link app
Property | Type | Default | Description | Example |
---|---|---|---|---|
name | String | DADI Web (Repo Default) |
The name of your application, used for the boot message | My project |
Anchor link server
Property | Type | Default | Description | Example |
---|---|---|---|---|
host | String | 0.0.0.0 |
The hostname or IP address to use when starting the Web server | example.com |
port | Number | 8080 |
The port to bind to when starting the Web server | 80 |
socketTimeoutSec | Number | 30 |
The number of seconds to wait before closing an idle socket | 10 |
protocol | String | http |
The protocol the web application will use | https |
redirectPort | Number | - | A port to redirect from, to the port |
80 |
sslPassphrase | String | - | The passphrase of the SSL private key | secretPassword |
sslPrivateKeyPath | String | - | The path to the SSL private key | /etc/ssl/key.pem |
sslCertificatePath | String | - | The filename of the SSL certificate | /etc/ssl/cert.pem |
sslIntermediateCertificatePath | String | - | The filename of an SSL intermediate certificate, if any | /etc/ssl/ca.pem |
sslIntermediateCertificatePaths | Array | - | The filenames of SSL intermediate certificates, overrides sslIntermediateCertificatePath (singular) |
[ '/etc/ssl/ca/example.pem', '/etc/ssl/ca/other.pem' ] |
Anchor link api
Property | Type | Default | Description | Example |
---|---|---|---|---|
host | String | 0.0.0.0 |
The hostname or IP address of the DADI API instance to connect to | api.example.com |
protocol | String | http |
The protocol to use | https |
port | Number | 8080 |
The port of the API instance to connect to | 3001 |
Alternatively you can specify an object of configuration objects if you intend to use multiple data provider configurations. For example:
"api": {
"main": {
"host": "127.0.0.1",
"port": 3000,
"auth": {
"tokenUrl": "/token",
"clientId": "your-client-id",
"secret": "your-secret"
}
},
"secondary": {
"host": "127.0.0.1",
"port": 3000,
"auth": {
"tokenUrl": "/token",
"clientId": "your-client-id",
"secret": "your-secret"
}
}
}
You can then reference the api
config in your datasource specification:
{
"datasource": {
"key": "articles",
"source": {
"api": "main",
"endpoint": "1.0/cloud/articles"
},
"count": 12,
"paginate": false
}
The defaults order for finding API information is:
- The
api
andauth
blocks - Or an
api
configuration defined with "type": "dadiapi" - Or an api with no defined type
- Or the api in position
api[0]
- Or settings in the source of the datasource itself
Anchor link auth
This block is used in conjunction with the
api
block above, but is also used in calls to the status and cache flush endpoints.
Property | Type | Default | Description | Example |
---|---|---|---|---|
tokenUrl | String | /token |
The endpoint to use when requesting Bearer tokens from DADI API | anotherapi.example.com/token |
protocol | String | http |
The protocol to use when connecting to the tokenUrl |
https |
clientId | String | your-client-key |
Should reflect what you used when you setup your DADI API | my-user |
secret | String | your-client-secret |
The corresponding password | my-secret |
Anchor link caching
N.B. Caching across DADI products is standardised by DADI Cache.
Property | Type | Default | Description | Example |
---|---|---|---|---|
ttl | Number | 300 |
The time, in seconds, after which cached data is considered stale | 3600 |
Anchor link directory
Property | Type | Default | Description | Example |
---|---|---|---|---|
enabled | Boolean | true |
If enabled, cache files will be saved to the filesystem | false |
path | String | ./cache/web |
Where to store the cache files | ./tmp |
extension | String | html |
The default file extension for cache files. Note that Web will override this if compression is enabled | json |
Anchor link redis
You will need to have a Redis server running to use this.
Property | Type | Default | Description | Example |
---|---|---|---|---|
enabled | Boolean | true |
If enabled, cache files will be saved to specified Redis server | false |
cluster | Boolean | false |
||
host | String | 127.0.0.1 |
||
port | Number | 6379 |
||
password | String | - |
Anchor link engines
In version 3.0 and above, DADI Web can handle multiple template engines, the default being a Dust.js interface. You can pass configuration options to these adaptors in this block.
Please see Views later for more information.
Anchor link headers
Property | Type | Default | Description | Example |
---|---|---|---|---|
useGzipCompression (deprecated, see useCompression ) |
Boolean | |||
useCompression | Boolean | true |
Attempts to compress the response, including assets, using either Brotli or Gzip | false |
cacheControl | Object | { 'image/png': 'public, max-age=86400', 'image/jpeg': 'public, max-age=86400', 'text/css': 'public, max-age=86400', 'text/javascript': 'public, max-age=86400', 'application/javascript': 'public, max-age=86400', 'image/x-icon': 'public, max-age=31536000000' } |
A set of custom cache-control headers (in seconds) for different content types |
In addition, a cacheControl
header can be used for a 301/302 redirect by adding to the configuration block:
"headers": {
"cacheControl": {
"301": "no-cache"
}
}
Anchor link logging
Property | Type | Default | Description | Example |
---|---|---|---|---|
enabled | Boolean | true |
If true, logging is enabled using the following settings. | false |
level | debug ,info ,warn ,error ,trace |
info |
The level at which log messages will be written to the log file. | warn |
path | String | ./log |
The absolute or relative path to the directory for log files | /data/app/log |
filename | String | web |
The filename to use for the log files. The name you choose will be given a suffix indicating the current application environment | my_application_name |
extension | String | log |
The extension to use for the log files | txt |
Anchor link accessLog
Property | Type | Default | Description | Example |
---|---|---|---|---|
enabled | Boolean | true |
If true, HTTP access logging is enabled. The log file name is similar to the setting used for normal logging, with the addition of 'access'. For example dadi-web.access.log |
false |
kinesisStream | String | - | An AWS Kinesis stream to write to log records to | web_aws_kinesis |
Anchor link aws
For use with the above block
logging.accessLog
.
Property | Type | Default | Description | Example |
---|---|---|---|---|
accessKeyId | String | |||
secretAccessKey | String | |||
region | String |
Anchor link rewrites
Property | Type | Default | Description | Example |
---|---|---|---|---|
datasource | String | - | The name of a datasource used to query the database for redirect records matching the current URL. More info | redirects |
path | String | - | The path to a file containing rewrite rules | workspace/routes/rewrites.txt |
forceLowerCase | Boolean | false |
If true, converts URLs to lowercase before redirecting | true |
forceTrailingSlash | Boolean | false |
If true , adds a trailing slash to URLs before redirecting |
true |
stripIndexPages | Array | - | A set of common index page filenames to remove from URLs | [‘index.php', 'default.aspx'] |
Anchor link global
The global
section can be used for any application parameters that should be available for use in page templates, such as asset locations, 3rd party account identifiers, etc
"global" : {
"baseUrl": "http://www.example.com"
}
In the above example baseUrl
would be available to a page template and could be used in the following way:
<html>
<body>
<h1>Welcome to DADI Web</h1>
<img src="{global.baseUrl}/images/welcome.png"/>
</body>
</html>
Anchor link globalEvents
Events to be loaded on every request.
Anchor link paths
Paths can be used to configure where any folder of the app assets are located.
For example:
"paths": {
"workspace": "workspace",
"datasources": "workspace/datasources",
"pages": "workspace/pages",
"events": "workspace/events",
"middleware": "workspace/middleware",
"media": "workspace/media",
"public": "workspace/public",
"routes": "workspace/routes"
}
Anchor link debug
See debugging
If set to true
, Web logs more information about routing, caching etc. Caching is also disabled.
Anchor link allowDebugView
See Debug view
If set, enabled the page debug view to be accessible by appending the querystring ?debug
to the end of any URL.
Anchor link Environmental variables
Best practice is to avoid keeping sensitive information inside a config.*.json
. Therefore anywhere a password or secret is used in a config file can be substituted for an environmental variable.
Variable | Block to substitute |
---|---|
AUTH_TOKEN_ID | auth.clientId |
AUTH_TOKEN_SECRET | auth.secret |
AWS_ACCESS_KEY | aws.accessKeyId |
AWS_SECRET_KEY | aws.secretAccessKey |
AWS_REGION | aws.region |
NODE_ENV | env |
REDIS_HOST | caching.redis.host |
REDIS_PORT | caching.redis.post |
REDIS_PASSWORD | caching.redis.password |
SESSION_SECRET | sessions.secret |
PORT | server.port |
PROTOCOL | server.potocol |
SSL_PRIVATE_KEY_PASSPHRASE | server.sslPassphrase |
SSL_PRIVATE_KEY_PATH | server.sslPrivateKeyPath |
SSL_CERTIFICATE_PATH | server.sslCertificatePath |
SSL_INTERMEDIATE_CERTIFICATE_PATH | server.sslIntermediateCertificatePath |
SSL_INTERMEDIATE_CERTIFICATE_PATHS | server.sslIntermediateCertificatePaths |
TWITTER_CONSUMER_KEY | twitter.consumerKey |
TWITTER_CONSUMER_SECRET | twitter.consumerSecret |
TWITTER_ACCESS_TOKEN_KEY | twitter.accessTokenKey |
TWITTER_ACCESS_TOKEN_SECRET | twitter.accessTokenSecret |
WORDPRESS_BEARER_TOKEN | workspress.bearerToken |
Anchor link Adding pages
A page on your website consists of two files within your workspace: a JSON specification and a template.
N.B. The location of this folder is configurable, but defaults to workspace/pages.
Anchor link Example specification
Here is an example page specification, with all options specified.
{
"page": {
"name": "People",
"description": "A page for displaying People records."
},
"settings": {
"cache": true,
"beautify": true,
"keepWhitespace": true,
"passFilters": true
},
"routes": [
{
"path": "/people"
}
],
"contentType": "text/html",
"template": "people.dust",
"datasources": [
"allPeople"
],
"requiredDatasources": [
"allPeople"
],
"events": [
"processPeopleData"
],
"preloadEvents": [
"geolocate"
]
}
Anchor link page
Property | Type | Default | Description |
---|---|---|---|
name | String | - | Used by the application for identifying the page internally. |
Any other properties you add are passed to the page data. Useful for maintaining HTML <meta>
tags, languages etc.
Anchor link settings
Property | Type | Default | Description |
---|---|---|---|
cache | Boolean | Reflects the caching settings in the main config file |
Used by the application for identifying the page internally. |
Anchor link routes
For every page added to your application, a route is created by default. A page’s default route is a value matching the page name. For example if the page name is books
the page will be available in the browser at /books
.
To make the books page reachable via a different URL, simply add (or modify) the page’s routes property:
"routes": [
{
"path": "/reading"
}
]
For detailed documentation of routing, see Routing.
Anchor link Content Type
The default content type is text/html
. This can be overridden by defining the contentType
in the root of the page config.
"contentType": "application/xhtml+xml"
Anchor link Templates
Template files are stored in the same folder as the page specifications and by default share the same filename as the page.json
. Unless the page specification contains an explicit template
property, the template name should match the page specification name.
See Views for further documentation.
Anchor link Datasources
An array containing datasources that should be executed to load data for the page.
For detailed documentation of datasources, see Datasources
"datasources": [
"datasource-one",
…
]
Anchor link Required Datasources
Allows specifying an array of datasources that must return data for the page to function. If any of the listed datasources return no results
, a 404 is returned. The datasources specified must exist in the datasources
array.
"requiredDatasources": [
"datasource-one",
...
]
Anchor link Events
An array containing events that should be executed after the page's datasources have loaded data.
"events": [
"event-one",
...
]
For detailed documentation of events, see Events
Anchor link Preload Events
An array containing events that should be executed before the rest of the page's datasources and events.
Preload events are loaded from the filesystem in the same way as a page's regular events, and a Javascript file with the same name must exist in the events
path.
"preloadEvents": [
"preloadevent-one",
...
]
Anchor link Caching
If true the output of the page will be cached using cache settings in the main configuration file.
"settings": {
"cache": true
}
For detailed documentation of page caching, see Caching.
Anchor link Routing, rewrites and redirects
Routing allows you to define URL endpoints for your application and control how Web responds to client requests.
Anchor link Basic Routing
Adding routes provides URLs for interacting with the application. A route specified as /contact-us
, for example, will make a URL available to your end users as http://www.example.com/contact-us
.
Anchor link Page Routing
For every page added to your application, a route
is created by default. A page's default route is a value matching the page name. For example if the page name is books
the page will be available in the browser at /books
.
To make the books
page reachable via a different URL, simply add (or modify) the page's routes
property:
{
"routes": [
{
"path": "/reading"
}
]
}
Anchor link Dynamic parameters
Routes may contain dynamic segments or named parameters which are resolved from the request URL and can be utilised by the datasources and events attached to the page.
A route segment with a colon at the beginning indicates a dynamic segment which will match any value. For example, a page with the route /books/:title
will be loaded for any request matching the format. DADI Web will extract the :title
parameter and add it to the req.params
object, making it available for use in the page's attached datasources and events.
The following URLs match the above route, with the segment defined by :title
extracted, placed into req.params
and accessible via the property title
.
URL | Named Parameter :title |
Request Parameters req.params |
---|---|---|
/books/war-and-peace | war-and-peace |
{ title: "war-and-peace" } |
/books/sisters-brothers | sisters-brothers |
{ title: "sisters-brothers" } |
Anchor link Optional Parameters
Parameters can be made optional by adding a question mark ?
.
For example the route /books/:page?
will match requests in both the following formats:
URL | Matched? | Named Parameters | Request Parameters req.params |
|
---|---|---|---|---|
/books | Yes | {} |
||
/books/2 | Yes | :page |
{ page: "2" } |
Anchor link Parameter Format
Specifying a format for a parameter can help Web identify the correct route to use. We can use the same example as above, where the URL has an optional page
parameter. If we add a regular expression to this parameter indicating that it should only match numbers, any URL that doesn't contain numbers in this segment will not match the route.
Example
The route /books/:page(\\d+)
will only match a URL that has books
in the first segment and a number in the second segment:
URL | Matched? | Named Parameters | Request Parameters req.params |
|
---|---|---|---|---|
/books/war-and-peace | No | |||
/books/2 | Yes | :page |
{ page: "2" } |
N.B. DADI Web uses the Path to Regexp library when parsing routes and parameters. More information on parameter usage can be found in the Github repository.
Anchor link Multiple Route Pages
The routes
property makes it easy for you to define "multiple route" pages, where one page specification can handle requests for multiple routes.
DADI Web versions >= 1.7.0
DADI Web 1.7.0 introduced a more explicit way of specifying multiple routes per page . The route
property has been replaced with routes
which should be an Array of route objects.
Each route object must contain, at the very least, a path
property. At startup, Web adds the value of each path
property to an internal collection of routes for matching incoming requests.
{
"routes": [
{
"path": "/movies/:title"
},
{
"path": "/movies/news/:title?/"
},
{
"path": "/movies/news/:page?/"
}
]
}
In the above example, the same page (and therefore it's template) will be loaded for requests matching any of the formats specified by the path
properties:
http://web.somedomain.tech/movies/deadpool
http://web.somedomain.tech/movies/news/
http://web.somedomain.tech/movies/news/2
http://web.somedomain.tech/movies/news/deadpool
Anchor link Route Priority
DADI Web sorts your routes into a priority order so that the most likely matches are easier to find.
- In Web, the most important parts of a route are the static segments, or rather the non-dynamic segments, for example
/books
. The more static segments in a route the higher its priority. - The second most important parts are the mandatory dynamic segments, for example
/:title
. - The least important parts are the optional dynamic segments, for example
/:year?
. - Any route with a
page
parameter gets a slight edge, with 1 point being added to its priority.
Path | Priority |
---|---|
/movies/news/:page(\\d+)?/ |
12 |
/movies/reviews/:page(\\d+)? |
12 |
/movies/features/:page(\\d+)?/ |
12 |
/movies/news/:title?/ |
11 |
/movies/features/:title?/ |
11 |
/movies/reviews/ |
10 |
/movies/:title/:page(\\d+)? |
9 |
/movies/:title/:content? |
8 |
/movies/ |
5 |
Anchor link Route Validation
An application may have more than one route that matches a particular URL, for example two routes that each have one dynamic segment:
/:genres
/:categories
In this case it is possible to provide DADI Web with some rules for determining the correct routes based on the parameters in the request. Parameter checks currently supported are:
preload
- tests the parameter value exists in a set of preloaded datain
- tests the parameter value exists in an array of static valuesfetch
- performs a datasource lookup using the parameter value as a filter
Anchor link Parameter Validation
Anchor link Preloaded data (preload
)
To validate parameters against preloaded data, you first need to configure Web to preload some data. Add a block to the main configuration file like the example below, using your datasource names in place of "channels":
{
"data": {
"preload": [
"channels"
]
}
}
{
"routes": [
{
"path": "/:channel/news/",
"params": [
{
"param": "channel",
"preload": {
"source": "channels",
"field": "key"
}
}
]
}
]
}
Anchor link Static array test (in
)
{
"routes": [
{
"path": "/movies/:title/:subPage?/",
"params": [
{
"param": "subPage",
"in": ["review"]
}
]
}
]
}
Anchor link Datasource lookup (fetch
)
{
"routes": [
{
"path": "/movies/:title/:content?/",
"params": [
{
"fetch": "movies"
}
]
}
]
}
Anchor link Route Constraint Functions
In the case of ambiguous routes it is possible to provide DADI Web with a constraint function to check each matching route against some business logic or existing data.
Returning
true
from a constraint instructs DADI Web that this is the correct route, the attached datasources and events should be run and the page displayed.
Returning
false
from a constraint instructs DADI Web to try the next matching route (or return a 404 if there are no further matching routes).
Constraints are added as a route property in the page specification file:
{
"routes": [
{
"path": "/:people",
"constraint": "nextIfNotPeople"
}
]
}
To add constraint functions, create a file in the routes
folder (by default configured as app/routes
). The file MUST be named constraints.js
.
In the following example the route has a dynamic parameter subPage
. The constraint function nextIfNewsOrFeatures
will check the value of the subPage
parameter and return false
if it matches "news" or "features", indicating to DADI Web that the next matching route should be tried (or a 404 returned if there are no further matching routes).
app/pages/movies.json
{
"routes": [
{
"path": "/movies/:subPage",
"constraint": "nextIfNewsOrFeatures"
}
]
}
app/routes/constraints.js
module.exports.nextIfNewsOrFeatures = function (req, res, callback) {
if (req.params.subPage === 'news' || req.params.subPage === 'features') {
return callback(false)
}
else {
return callback(true)
}
}
}
Anchor link Constraint Datasources
Note: Deprecated in Version 1.7.0
An existing datasource can be used as the route constraint. The specified datasource must exist in datasources
(by default configured as app/datasources
). The following examples have some missing properties for brevity.
app/pages/books.json
{
"route": {
"paths": ["/:genre"],
"constraint": "genres"
}
}
app/datasources/genres.json
{
"datasource": {
"key": "genres",
"name": "Genre datasource",
"source": {
"endpoint": "1.0/library/genres"
},
"count": 1,
"fields": { "name": 1, "_id": 0 },
"requestParams": [
{ "param": "genre", "field": "title" }
]
}
}
In the above example a request for http://www.example.com/crime
will call the genres
datasource, using the requestParams
to supply a filter to the endpoint. The request parameter :genre
will be set to crime
and the resulting datasource endpoint will become:
/1.0/library/genres?filter={"title":"crime"}
If there is a result for this datasource query, the constraint will return true
, otherwise false
.
Anchor link Template URL Building
Using toPath()
:
var app = require('dadi-web');
var page = app.getComponent('people');
var url = page.toPath({ id: '1234' });
"/person/1234"
Anchor link Using Request Parameters
See Datasource Specification for more information regarding the use of named parameters in datasource queries.
Anchor link URL Rewriting and Redirects
Anchor link forceLowerCase
With this property set to true
, Web converts incoming URLs to lowercase and sends a 301 Redirect to the browser with the lowercased version of the URL.
{
"forceLowerCase": true
}
Anchor link forceTrailingSlash
With this property set to true
, Web adds a trailing slash to incoming URLs and sends a 301 Redirect to the browser with the new version of the URL.
{
"forceTrailingSlash": true
}
Anchor link stripIndexPages
This property accepts an array of filenames to remove from URLs. Useful for when you're migrating from another system and search engines may have indexed URLs containing legacy files. For example http://legacy-web.example.com/index.php
{
"stripIndexPages": ["index.php", "default.aspx"]
}
Anchor link URL Rewriting
URL rewriting support in DADI Web is similar to Apache's mod_rewrite module. To setup, add your rewrite rules to a text file, one rule per line, and update the Web configuration with the path to your file:
main configuration file
"rewrites": {
"path": "workspace/routes/rewrites.txt"
}
Anchor link Rewrite Rules
Each rewrite rule consists of three sections: a match, a replacement and a set of flags. The "match" section allows for regular expression matching of request URLs. If a match is found, the request is modified to use the contents of the "replacement" section. The "flags" modify the behaviour of the match or the final request.
Anchor link Defined parameters
Defined parameters can be wrapped with ()
and then accessed in the replacement with $n
. For example, if migrating to a new URL structure that no longer has the "blog/" portion, it is possible to redirect all legacy URLs to the new structure using the following rule. The only defined parameter is "everything" following the "blog/" part of the URL, and we're asking Web to use only the contents of that parameter as the replacement:
^/blog/(.*)$ /$1 [L]
Will redirect /blog/2017/06/10/xyz
to /2017/06/10/xyz
^/blog/(.*)$ /$1 [L]
Examples
1) Send a HTTP 301 redirect from /books/hemingway
to /books?author=hemingway
^/books/(.*)$ /books?author=$1 [R=301,L]
2) Add a trailing slash to any URL if it doesn't have one already, sending a HTTP 301 redirect with the new URL
^(.*[^/])$ $1/ [R=301,L]
Anchor link Flags
Flags modify the behaviour of the URL rewriting system. Put the required flags within []
and separate them with commas.
- Last
[L]
: if a path matches, any subsequent rewrite rules will be disregarded - Proxy
[P]
: proxy your requests^/test/proxy/(.*)$ http://nodejs.org/$1 [P]
- Redirect
[R]
, [R=301]`: issue a redirect for the request - Nocase
[NC]
: regex match will be case-insensitive - Forbidden
[F]
: issue a HTTP 403 Forbidden response - Gone
[G]
: issue a HTTP 410 Gone response - Type
[T=*]
: sets the content-type to the specified one (replace * with mime-type) - Host
[H]
,[H=*]
: matches on the request host header (replace * with a regular expression that matches a hostname)
For more info about available flags, please see the Apache page.
Anchor link Redirecting domains
To redirect from one domain to another, for example to remove www.
from requests and redirect them to the root domain, add a rule similar to the following. The Host flag is used, specifying the request host header that this rule should match. The "match" section specifies a single defined parameter which is used in the "replacement" section, appended to the hardcoded new domain.
\/(.*)$ https://dadi.tech/$1 [H=www\.dadi\.tech, R=302,NC,L]
Anchor link Views
Anchor link Engines
You can use a variety of template engines with DADI Web. We maintain several template engines, such as Dust, Pug and Handlebars. You can find more on NPM.
Anchor link Install a new engine
Each package lists it's own install instructions, but they all follow the same pattern:
Install the interface you want:
npm install @dadi/web-handlebars --save
Edit your app entry file (by default this is server.js
):
require('@dadi/web')({
engines: [
require('@dadi/web-handlebars')
]
})
Anchor link Creating your own engine
Full instructions for this are available in our Web sample engine repo.
Anchor link Error pages
DADI Web has default error pages, but it will look for templates in the pages
folder which match the error code needing to be expressed e.g., 404.dust
.
Anchor link Sessions
DADI Web uses the express-session library to handle sessions. Visit that project's homepage for more detailed information regarding session configuration.
Anchor link Session Configuration
Note: Sessions are disabled by default. To enable them in your application, add the following to your configuration file:
"sessions": {
"enabled": true
}
A full configuration block for sessions contains the following properties:
"sessions": {
"enabled": true,
"name": "your-cookie-name"
"secret": "your-secret-key",
"resave": false,
"saveUninitialized": false,
"store": "",
"cookie": {
"maxAge": 60000,
"secure": false
}
}
Anchor link Session Configuration Properties
Property | Description | Default |
---|---|---|
enabled | If true , sessions are enabled. |
false |
name | The session cookie name. | "dadiweb.sid" |
secret | The secret used to sign the session ID cookie. This can be either a string for a single secret, or an array of multiple secrets. If an array of secrets is provided, only the first element will be used to sign the session ID cookie, while all the elements will be considered when verifying the signature in requests. | "dadiwebsecretsquirrel" |
resave | Forces the session to be saved back to the session store, even if the session was never modified during the request. | false |
saveUninitialized | Forces a session that is "uninitialized" to be saved to the store. A session is uninitialized when it is new but not modified. Choosing false is useful for implementing login sessions, reducing server storage usage, or complying with laws that require permission before setting a cookie. Choosing false will also help with race conditions where a client makes multiple parallel requests without a session. | false |
store | The session store instance, defaults to a new MemoryStore instance. | The default is an empty string, which uses a new MemoryStore instance. To use MongoDB as the session store, specify a MongoDB connection string such as "mongodb://host:port/databaseName" or "mongodb://username:password@host:port/databaseName" if your database requires authentication. To use Redis, specify a Redis server's address and port number: "redis://127.0.0.1:6379" . |
cookie | ||
cookie.maxAge | Set the cookie’s expiration as an interval of seconds in the future, relative to the time the browser received the cookie. null means no 'expires' parameter is set so the cookie becomes a browser-session cookie. When the user closes the browser the cookie (and session) will be removed. |
60000 |
cookie.secure | HTTPS is necessary for secure cookies. If secure is true and you access your site over HTTP, the cookie will not be set. |
false |
Anchor link Using the session
Session data can easily be accessed from an event or custom middleware.
const Event = (req, res, data, callback) => {
if (req.session) {
req.session.someProperty = "some value"
req.session.save(function (err) {
// session saved
})
data.session_id = req.session.id
}
callback(null, data)
}
Anchor link Adding data
Anchor link Datasources
Datasources are used to connect to both internal and external data providers to load data for rendering pages.
my-web/
app/
datasources/
books.json # a datasource specification
events/
pages/
Anchor link Datasource Specification File
{
"datasource": {
"key": "books",
"name": "Books datasource",
"source": {
"type": "dadiapi",
"endpoint": "1.0/library/books"
},
"paginate": true,
"count": 5,
"sort": { "name": 1 },
"filter": {},
"fields": {}
}
}
Anchor link Datasource Configuration
Property | Description | Default value | Example | |
---|---|---|---|---|
key | Short identifier of the datasource. This value is used in the page specification files to attach a datasource | "books" |
||
name | This is the name of the datasource, commonly used as a description for developers | "Books" |
||
paginate | true | true | ||
count | Number of items to return from the endpoint per page. If set to '0' then all results will be returned, up to the limit specified in the collection schema settings block |
20 | 5 | |
sort | A JSON object with fields to order the result set by | {} // unsorted |
{ "title": 1 } // sort by title ascending , { "title": -1 } // sort by title descending |
|
filter | A JSON object containing a MongoDB query | { "SaleDate" : { "$ne" : null} } |
||
filterEvent | An event file to execute which will generate the filter to use for this datasource. The event must exist in the configured events path | "getBookFilter" |
||
fields | Limits the fields to return in the result set | { "title": 1, "author": 1 } |
||
requestParams | An array of parameters the datasource can obtain from the querystring, config or the session. See Passing Parameters for more. | [ { "param": "author", "field": "author_id" } ] |
||
source | ||||
type | (optional) Determines whether the data is from a remote endpoint or local, static data | "remote" |
"remote" , "static" |
|
protocol | (optional) The protocol portion of an endpoint URI | "http" |
"http" , "https" |
|
host | (optional) The host portion of an endpoint URL | The main config value api.host |
"api.somedomain.tech" |
|
port | (optional) The port portion of an endpoint URL | The main config value api.port |
3001 |
|
endpoint | The path to the endpoint which contains the data for this datasource | "/1.0/news/articles" |
||
caching | ||||
enabled | Sets caching enabled or disabled | false |
true |
|
ttl | ||||
directory | The directory to use for storing cache files, relative to the root of the application | "./cache" | ||
extension | The file extension to use for cache files | "json" | ||
auth | ||||
type | "bearer" |
|||
host | "api.somedomain.tech" |
|||
port | 3000 |
|||
tokenUrl | "/token" |
|||
credentials | { "clientId": "your-client-key", "secret": "your-client-secret" } |
Anchor link Passing parameters
requestParams
is an array of parameters that can be used to generate a datasource filter or even modify a datasource's endpoint. Web can obtain parameters from the request querystring, the configuration or from the session (if one exists).
The syntax for request parameters in a datasource specification looks like the following:
"requestParams": [
{
"source": "config|session|request",
"param": "param",
"field": "field",
"target": "filter|endpoint"
}
]
Property | Description | Default |
---|---|---|
source | Where to look for the parameter specified by "param" | "request" |
param | The parameter or property that contains the value to be used | |
field | The name of the property to add to a filter, or the endpoint placeholder to replace | |
target | Whether this parameter should be added to a datasource filter, or directly into the endpoint | "filter" |
Anchor link Extract parameters from the request (default)
The simplest usecase for requestParams
is in conjunction with dynamic parameters obtained from route
properties in a page specification.
For example, given a collection books
with the fields _id
, title
With the page route /books/:title
and the below datasource specification, Web will extract the :title
parameter from the URL and use it to query the books
collection using the field title
.
With a request to http://www.somedomain.tech/books/sisters-brothers, the named parameter :title
is sisters-brothers
. A filter query is constructed for the datasource using this value.
The resulting query passed to the underlying datastore will be: { "title" : "sisters-brothers" }
See Routing for detailed routing documentation.
Page specification
"routes": [{
"path": "/books/:title"
}]
Datasource specification
"source": {
"endpoint": "1.0/library/books"
},
"requestParams": [
{
"param": "title",
"field": "title"
}
]
In this case, the requestParam'sparam
property is being used as the field to filter on and so must match a named parameter in the page specification'sroutes
.
Anchor link Extract parameters from config
Values held in the configuration file can be used to replace placeholders in an endpoint. In the configuration below, a global
section has been added, containing a news item to retrieve from Hacker News.
config/config.development.json
{
"server": {
"host": "0.0.0.0",
"port": 3001
},
"global": {
"newsItems": {
"hackernewsItemId": "17200415"
}
}
}
With a Hacker News datasource, we can add a placeholder to the HN API endpoint {item}
and have Web replace that with the value obtained from the configuration file:
{
"datasource": {
"key": "hackernews",
"name": "Get specific news item",
"source": {
"type": "remote",
"protocol": "https",
"host": "hacker-news.firebaseio.com",
"port": 443,
"endpoint": "v0/item/{item}.json"
},
"requestParams": [
{
"source": "config",
"param": "global.newsItems.hackernewsItemId",
"field": "item",
"target": "endpoint"
}
]
}
}
If you've forgotten to add the configuration property, Web will throw the following error:
[2018-06-01 13:14:43.444] [LOG] Error: cannot find configuration param 'global.newsItems.hackernewsItemId'
✖ Error: cannot find configuration param 'global.newsItems.hackernewsItemId'
If configured successfully, Web will generate a request for the URL https://hacker-news.firebaseio.com:443/v0/item/17200415.json
:
03:18:39.020Z INFO dadi-web: GOT datasource "hackernews": https://hacker-news.firebaseio.com:443/v0/item/17200415.json (HTTP 200, 223 Bytes) (module=remote)
The resulting response will be added to the data context using the datasource's key (in this case hackernews
):
"hackernews": {
"by": "captn3m0",
"descendants": 1,
"id": 17200415,
"kids": [
17200762
],
"score": 4,
"time": 1527801802,
"title": "Show HN: OPML generator for following your starred GitHub project releases",
"type": "story",
"url": "https://opml.bb8.fun/"
}
Anchor link Extract parameters from session
Using a similar method to extracting parameters from the configuration, Web also supports obtaining values from the session if sessions are enabled. Simply set the source
property to "session"
and the param
property to the path to the required value:
{
"datasource": {
"key": "userDetails",
"source": {
"endpoint": "1.0/users/{userId}"
},
"requestParams": [
{
"source": "session",
"param": "user.profile._id",
"field": "userId",
"target": "endpoint"
}
]
}
}
Anchor link Extract parameters from chained datasource
In addition, chained datasources can now use data from the datasource they're attached to, modifying the endpoint if "target": "endpoint"
:
{
"datasource": {
"key": "chained-endpoint",
"source": {
"endpoint": "1.0/makes/{name}"
},
"chained": {
"datasource": "global", // the datasource we're waiting on
"outputParam": {
"param": "results.0.name", // where to find the value in the "global" datasource
"field": "name", // the value to replace in this endpoint URL
"target": "endpoint"
}
}
}
}
Anchor link Building filters
Filter Events can be used to generate filters for datasources. They are like regular Events but are designed to return a filter before the datasource is executed. See Filter Events for more information.
Anchor link Chained datasources
It is often a requirement to query a datasource using data already loaded by another datasource. DADI Web supports this through the use of "chained" datasources. Chained datasources are executed after all non-chained datasources, ensuring the data they rely on has already been fetched.
Add a chained
property to a datasource to make it reliant on data loaded by another datasource. The following datasource won't be executed until data from the books
datasource is a available:
{
"datasource": {
"key": "books-by-author",
"source": {
"type": "dadiapi",
"endpoint": "1.0/library/authors"
},
"chained": {
"datasource": "books" // the primary (non-chained) datasource that this datasource relies on
}
}
}
Chained datasources are not automatically added to the page. You must still include the datasource in the datasources block of your page config.
There are two ways to use query a chained datasource using previously-fetched data. One is Filter Generation and the other is Filter Replacement.
Anchor link Filter Generation
Filter Generation is used when the chained datasource currently has no filter, and it is relying on the primary datasource to provide its values.
Example: query the "authors" datasource, using the _id
from the "books" datasource
"chained": {
"datasource": "books",
"outputParam": {
"field": "_id", // the filter key to use for this datasource
"param": "results.0.author_id" // the path to the value this datasource will use in it's filter
}
}
Specifying a field
and a param
causes DADI Web to generate a filter for this datasource using values from the primary datasource. For example:
Results from primary datasource
{
"results": [
{
"fullName": "Ernest Hemingway",
"author_id": 1234567890
}
]
}
Generated filter for chained datasource
{ "_id": 1234567890 }
Anchor link Filter Replacement
Filter Replacement allows more advanced filtering and can inject a query into an existing datasource filter.
Using the query
property, Web extracts the specified value from the primary datasource (using the path from param
) and injects it into the query where {param}
has been left as a placeholder.
Next, Web takes the updated value of the query
property and injects the whole thing into the current datasource's filter
where it finds a placeholder matching the key of the chained datasource (in the example below, "{books}"
is the placeholder).
"filter": ["{books}",{"$group":{"_id":{"genre":"$genre"}}}],
"chained": {
"datasource": "books",
"outputParam": {
"param": "results.0.genre_id",
"type": "Number",
"query": {"$match":{"genre_id": "{param}"}}
}
}
Anchor link Chained datasource configuration
Property | Description | Example | |
---|---|---|---|
datasource | Should match the key property of the primary datasource. |
||
outputParam | |||
param | The param value specifies where to locate the output value in the results returned by the primary datasource. |
"results.0._id" |
|
field | The field value should match the MongoDB field to be queried. |
"id" |
|
type | The type value indicates how the param value should be treated (currently only "Number" is supported). |
"Number" |
|
query | The query property allows more advanced filtering, see below for more detail. |
{} |
Anchor link Chained datasource full example
On a page that displays a car make and all it's associated models, we have two datasources querying two collections, makes and models.
Collections
- makes has the fields
_id
andname
- models has the fields
_id
,makeId
andname
Datasources
- The primary datasource,
makes
(some properties removed for brevity)
{
"datasource": {
"key": "makes",
"source": {
"endpoint": "1.0/car-data/makes"
},
filter: { "name": "Ford" }
}
}
The result of this datasource will be:
{
"results": [
{
"_id": "5596048644713e80a10e0290",
"name": "Ford"
}
]
}
To query the models collection based on the above data being returned, add a chained
property to the models datasource specifying makes
as the primary datasource:
{
"datasource": {
"key": "models",
"source": {
"endpoint": "1.0/car-data/models"
},
"chained": {
"datasource": "makes",
"outputParam": {
"param": "results.0._id",
"field": "makeId"
}
}
}
}
In this scenario the models collection will be queried using the value of _id
from the first document of the results
array returned by the makes datasource.
If your query parameter must be passed to the endpoint as an integer, add a type
property to the outputParam
specifying "Number"
.
{
"datasource": {
"key": "models",
"source": {
"endpoint": "1.0/car-data/models"
},
"chained": {
"datasource": "makes",
"outputParam": {
"param": "results.0._id",
"type": "Number",
"field": "makeId"
}
}
}
}
Anchor link Preloading data
Web can be configured to preload data before each request. Add a block to the main configuration file like the example below, using your datasource names in place of "channels":
"data": {
"preload": [
"channels"
]
}
Anchor link Accessing preloaded data
const Preload = require('@dadi/web').Preload
const data = Preload().get('key')
Anchor link Data Providers
Loading data into the context for rendering requires a datasource. Each datasource specifies what data provider to use and any additional parameters that the data provider needs to retrieve the data.
Built-in data providers include:
- DADI API: retrieve data from an existing DADI API
- Rest API: retrieve data from miscellaneous REST APIs requiring authentication
- Remote: retrieve data from miscellaneous REST APIs
- Markdown: load data from a folder of Markdown files (or any other text file type)
- Twitter: retrive data from the Twitter API
- Wordpress: retrive data from a Wordpress API
Anchor link DADI API
Previous to 2.0 the datasource source type for connecting to a DADI API was called remote
. This was changed to dadiapi
to ensure clarity with the updated and repurposed Remote provider.
A typical datasource specification file would now contain the following:
"source": {
"type": "dadiapi",
"endpoint": "1.0/articles"
}
The default is dadiapi
, so there is no requirement to specify this property when connecting to a DADI API.
Anchor link Remote
Connect to a miscellaneous API via HTTP or HTTPS. See the following file as an example:
{
"datasource": {
"key": "instagram",
"name": "Grab instagram posts for a specific user",
"source": {
"type": "remote",
"protocol": "http",
"host": "instagram.com",
"endpoint": "{user}/?__a=1"
},
"auth": false,
"requestParams": [
{
"param": "user",
"field": "user",
"target": "endpoint"
}
]
}
}
Anchor link Rest API
Connect to any RestAPI, including one which requires authentication:
{
"datasource": {
"key": "twitter",
"source": {
"type": "restapi",
"provider": "twitter",
"endpoint": "statuses/show",
"auth": {
"oauth": {
"consumer_key": "xxx",
"consumer_secret": "xxx",
"token": "xxx",
"token_secret": "xxx"
}
}
},
"fields": {
"text": 1,
"user.screen_name": 1
},
"requestParams": [
{
"param": "tweetid",
"field": "id",
"target": "query"
}
]
}
}
provider
can be any of the @purest/providers. For example Facebook
, google
, twitter
.
The
source
section can be partly defined in the main configapi
block to save repetition if needed.
Anchor link Markdown
Serve content from a local folder containing text files. You can also specify the extension to grab. Web will process any Markdown formatting (with Marked) it finds automatically as well as any Jekyll-style front matter found. Any dates/times found will be processed through JavasScript’s Date()
function.
{
"datasource": {
"source": {
"type": "markdown",
"path": "./workspace/posts",
"extension": "md"
}
}
}
workspace/posts/somefolder/myslug.md
--
date: 2016-02-17
title: Your title here
--
Some *markdown*
When loaded becomes the following data:
{
"attributes": {
"date": "2016-02-17T00:00:00.000Z",
"title": "Your title here",
"_id": "myslug",
"_ext": ".md",
"_loc": "workspace/posts/somefolder/myslug.md",
"_path": [
"somefolder"
]
},
"original": "--\ndate: 2016-02-17\ntitle: Your title here\n--\nSome *markdown*",
"contentText": "Some *markdown*",
"contentHtml": "<p>Some <em>markdown</em></p>\n"
}
NB. _path
will exclude the datasource source.path
.
Anchor link RSS
You can grab an XML feed and access it in your templates like any other JSON source. For example:
{
"datasource": {
"key": "rss",
"name": "RSS",
"source": {
"type": "rss",
"endpoint": "https://github.com/dadi/web/releases.atom"
},
"count": 1,
"fields": {
"description": 1,
"pubDate": 1
}
}
}
Anchor link Adding logic
Anchor link Events
Events are server side JavaScript functions that can add additional functionality to a page. Events can serve as a useful way to implement logic in a logic-less template.
A simple use case for Events is counting how many users have clicked on a 'Like' button. To achieve this an Event file needs to be attached to the page which contains the 'Like' button. The Event file would check the POST request body contains expected values and then perhaps increase a counter stored in a database.
The How To Guides contains an example of an Event being used to send email via SendGrid in response to user interaction.
Anchor link The Event system
The Event system in DADI Web provides developers with a way to perform tasks related to the current request, end the current request or extend the data context that is passed to the rendering engine.
- Events are executed in sequence after a page's datasources have all returned, and they have access to the data loaded by all the datasources
- Events are attached to pages in the page specification file, using the
"events"
array
my-web/
workspace/
datasources/
events/
addAuthorInformation.js # an Event file
pages/
books.json
workspace/pages/books.json
"events": [
"addAuthorInformation"
]
An Event is declared in the following way, receiving the original HTTP request, the response, the data context and a callback function to return control back to the controller that called it:
workspace/events/example.js
const Event = function (req, res, data, callback) {
callback(null)
}
module.exports = function (req, res, data, callback) {
return new Event(req, res, data, callback)
}
Anchor link The data context
The data
argument that an Event receives is JSON which is eventually passed to the template rendering engine once all the Datasources and Events have finished running. data
may contain some or all of the following:
- metadata about the current page
- request parameters
- data loaded by all datasources that have been run before the Events started executing
- data added by previous Events
- global configuration settings
Anchor link Sample Event file
workspace/events/full-example.js
/**
* <Event Description>
*
* @param {IncomingMessage} req - the original HTTP request
* @param {ServerResponse} res - the HTTP response
* @param {object} data - contains the data already loaded by the page's datasources and any previous events that have fired
* @param {function} callback - call back to the controller with two arguments, `err` and the result of the event processing
*/
const Event = (req, res, data, callback) => {
let result = {}
if (data.hasResults('books')) {
result = {
title: data.books.results[0].title
}
}
else {
result = {
title: "Not found"
}
}
// return a null error and the result
callback(null, result)
}
module.exports = function (req, res, data, callback) {
return new Event(req, res, data, callback)
}
module.exports.Event = Event
Anchor link Global Events
In addition to attaching Events to specific pages in the application, it is possible to declare "global events" that are fired for every page. Global Events are fired at the beginning of the request cycle, before datasources and other page Events. Add a "globalEvents" section to the main configuration file:
globalEvents: [
"eventName"
]
Anchor link Preload Events
Preload Events are similar to Global Events, in that they are fired at the beginning of the request cycle, before any data is loaded from datasources. To attach a Preload Event to a page, add a "preloadEvents" block to the page specification file:
"preloadEvents": [
"preloadevent-one"
]
Anchor link Filter Events
Filter Events can be used to generate filters for datasources. They are like regular Events but are designed to return a filter to the datasource so it can query it's underlying source. This could be useful when needing to specify a filter for a datasource that relies on parameters that can't be determined from the request parameters (that is, req.params
).
Any filter already specified by the datasource specification will be extended with the result of the filter event.
A filter event can be attached to a datasource specification using the property filterEvent
. That value must match the filename of an existing event file, without it's extension:
workspace/datasources/books.json
{
"datasource": {
"key": "books",
"source": {
"type": "dadiapi",
"endpoint": "1.0/library/books"
},
"count": 10,
"sort": {},
"filter": {"borrowed": true},
"filterEvent": "injectCurrentDate",
"fields": ["title", "author"]
}
}
workspace/events/injectCurrentDate.js
const Event = function (req, res, data, callback) {
const filter = { date: Date.now() }
// call the callback function, passing null in the error positon
callback(null, filter)
}
module.exports = function (req, res, data, callback) {
return new Event(req, res, data, callback)
}
With the above examples, the datasource instance will be modified as follows. The filter
property will be extended to add a date
property (from the filter event), and a new filterEventResult
property is added which contains the result of executing the filter event:
filter: { borrowed: true, date: 1507566199527 },
filterEventResult: { date: 1507566199527 }
Anchor link Middleware
Middleware functions are functions that can be added to your Web application and executed in sequence for each request. Each function has access to the request object (req
),
the response object (res
), and the next middleware function in the stack (by convention, a variable named next
).
Middleware functions can:
- execute any code
- modify the request and response objects
- end the request-response cycle (by calling
res.end()
) - call the next middleware function in the stack (by calling
next()
)
Note: if the currently executing middleware function does not end the request-response cycle, it must call next()
to pass control to the next middleware function. Failure to do so will cause the request to hang.
Middleware functions are stored as JavaScript files in your application's middleware
folder. The location of this folder is configurable but defaults to workspace/middleware
.
your-project/
config/
workspace/
datasources/ # datasource specifications
events/ # event files
middleware/ # middleware files
log.js # middleware file
pages/ # page specifications
A DADI Web application can use the following types of middleware:
- Application-level middleware
- Error-handling middleware
- Built-in middleware
- Third-party middleware
Anchor link Application-level middleware
A single middleware file may contain multiple functions, or they can be split across multiple files.
You bind application-level middleware functions to the instance of the app
object by using the app.use()
function, optionally specifying a route that determines the requests it applies to.
A middleware function with no route will be executed on every request.
Anchor link Route-less middleware functions
The following example shows a middleware function with no route. The function is executed every time the application receives a request:
const Middleware = function (app) {
// output the time for each request
app.use((req, res, next) => {
console.log('Request received at:', Date.now())
next()
})
}
module.exports = function (app) {
return new Middleware(app)
}
module.exports.Middleware = Middleware
Anchor link Route-specific middleware functions
The following example shows a middleware function mounted at the /users
route. The function is only executed for requests to the /users
route.
const Middleware = function (app) {
// output the time for each request
app.use('/users', (req, res, next) => {
console.log('Request received at:', Date.now())
next()
})
}
module.exports = function (app) {
return new Middleware(app)
}
module.exports.Middleware = Middleware
Anchor link HTTP method restriction
To restrict a middleware function to only certain HTTP methods, you can test the current request's method and call next()
if it doesn't match:
app.use('/users', (req, res, next) => {
if (req.method.toLowercase() !== 'get') {
return next()
}
console.log('GET request received at:', Date.now())
return next()
})
Anchor link Error-handling middleware
Error-handling middleware functions must accept four arguments. Without the additional argument (err
) the function will be interpreted as regular middleware and won't handle errors.
Define error-handling middleware functions in the same way as other middleware functions, except with four arguments instead of three:
app.use((err, req, res, next) => {
console.error(err.stack || err)
res.end(500, 'Server error!')
})
Anchor link Built-in middleware
DADI Web has some built-in middleware functions, which in some cases can be turned on or off using the main configuration file.
Type | Description |
---|---|
Body parser | parses the body of an incoming request and makes the data available as the property req.body |
Caching | determines if the current request can be handled by a previously cached response |
Compression | compresses the response before sending |
Request logging | logs every request to a file |
Sessions | handles session data |
Static files | serves static assets from the public folder, such as JavaScript, CSS, HTML files, images, etc |
Virtual directories | serves content from configured directories not handled by the existing page/route specifications |
Note: the body parser middleware can handle JSON, raw, plain text and URL-encoded request bodies. It does not handle multipart bodies due to their complex and typically large nature. For multipart bodies, try one of the following modules: busboy, multiparty, formidable or multer
Anchor link Third-party middleware
You can add third-party middleware to your DADI Web application to add new functionality that DADI Web doesn't have built-in. Simply install the Node.js module for the required functionality and load it in an application-level middleware function.
The following example shows how to use the module online to track online user activity using Redis:
$ npm install online
$ npm install redis
const Online = require('online')
const redis = require('redis')
const redisClient = redis.createClient()
// use the Redis client
const online = Online(redisClient)
const Middleware = function (app) {
// executes for every request
app.use((req, res, next) => {
// add a call to track current user's activity, assuming a `user` object in the request
online.add(req.user.id, (err) => {
if (err) {
return next(err) // call an error-handling middleware function
}
next() // calls the next middleware function (defined below)
})
})
// executes for every request
app.use((req, res, next) => {
// get users active in the last 10 minutes
online.last(10, (err, ids) => {
if (err) {
return next(err) // call an error-handling middleware function
}
console.log('Users online:', ids)
// call next middleware function
next()
})
})
}
module.exports = function (app) {
return new Middleware(app)
}
module.exports.Middleware = Middleware
Anchor link Middleware template
/**
* workspace/middleware/example.js
*/
const Middleware = function (app) {
// execute for every request
app.use((req, res, next) => {
console.log(req.url)
console.log(req.params)
next()
})
// route-mounted, execute for requests to /channel
app.use('/channel', (req, res, next) => {
console.log('Channel params:', JSON.stringify(req.params))
next()
})
// error handler
app.use((err, req, res, next) => {
console.log('Error:', err)
next()
})
}
module.exports = function (app) {
return new Middleware(app)
}
module.exports.Middleware = Middleware
Anchor link Post processors
Web "Post Processors" give the ability to manipulate the raw output of the template engine, before the page is cached and served. Similar to the Wordpress feature, it is useful for linting (thinking RSS feeds in particular), minifying and formatting.
Post processors are located in the workspace folder. The default location is workspace/processors
, however this is configurable using the paths.processors
configuration setting.
Post processors can be defined globally for the application or on a page-by-page basis. They are loaded in order of definition, globally defined ones first, and the output from each is passed to the next so you can chain functions.
Anchor link Example usage: minifying HTML
Anchor link Add globalPostProcessors
to configuration file
"globalPostProcessors": [
"minify-html"
]
Anchor link Add processor file
This processor uses the package html-minifier
/workspace/processors/minify-html.js
const minify = require('html-minifier').minify
module.exports = (data, output) => {
return minify(output, {
collapseWhitespace: true,
minifyCSS: true,
minifyJS: true,
removeRedundantAttributes: true,
useShortDoctype: true,
removeAttributeQuotes: true
})
}
Note:
- the page json data object is also passed to the function
- in addition it can be defined or disabled at page level in the
page.settings
block of any page specification file -- advice
Anchor link Page-specific processors
To add a post processor to a page add a postProcessors
property to the page's settings
block. Page-specific post processors are run after globally defined post processors.
"settings": {
"postProcessors": ["remove-swear-words"]
}
Anchor link Disabling post processors for a page
To disable post processors for a page add a postProcessors
property to the page's settings
block, and set this to false
.
"settings": {
"postProcessors": false
}
Anchor link Upgrading from Web 4.0
Previous versions of Web had a built-in post processor: beautify-html
. It was configured using the page configuration option page.settings.beautify
. To enable the same functionality in Web 5.0 and greater, follow the guide below.
Anchor link Install beautify package
npm install js-beautify
Anchor link Add new post processor file workspace/processors/beautify-html.js
const beautifyHtml = require('js-beautify').html
module.exports = (data, output) => {
return beautifyHtml(output)
}
Anchor link Modify page configuration
existing page.json
file example
{
"page": {
"name": "Sitemap page",
"description": "Sitemap",
"language": "en"
},
"settings": {
"cache": true,
"beautify": true
},
...
}
new page.json
file example
{
"page": {
"name": "Sitemap page",
"description": "Sitemap",
"language": "en"
},
"settings": {
"cache": true,
"postProcessors": [
"beautify-html"
]
},
...
}
Anchor link Internationalization
Adding support for internationalization in your application is relatively easy when using DADI API datasources to fetch data. API 4.2 (and above) has built-in support for documents containing fields with multiple language variations.
You must be using API 4.2 or above for this feature. See the documentation for API 4.2 for more detailed information on internationalization.
Anchor link Requesting a specific language
To retrieve your content in a specific language in Web, you simply need to modify your datasource to include a lang
parameter in the query string. For example, the datasource endpoint /1.0/library/books?lang=fr
will return data from API with the French versions of fields, if they exist:
{
"_id": "58176e72bafa53b625aebd4f",
"_i18n": {
"title": "fr",
"author": "en"
},
"title": "Le Petit Prince",
"author": "Antoine de Saint-Exupéry"
}
Note that any fields that don't have a translation matching the specified
lang
parameter will be returned in the default language.
Anchor link Dynamically selecting a language
To dynamically select a language parameter to send to API, it is possible to pass a parameter from the URL. There are two options for this: using a dynamic URL parameter or using the query string.
In both cases, when using the DADI API provider in a datasource, the value of the lang
parameter will be automatically added to the datasource's endpoint when making the request.
The
lang
parameter is added automatically; if you don't want to pass thelang
parameter you can explicitly disable it using thei18n
property in the datasource specification:
{ "datasource": { "key": "articles", "source": { "type": "dadiapi", "endpoint": "1.0/library/articles" }, "count": 4, "paginate": true, "i18n": false } }
Anchor link Dynamic URL parameter
In this case, you would have set up a page route to include a lang
portion, for example:
{
"page": {
"key": "article"
},
"datasources": [
"articles"
],
"routes": [
{
"path": "/:lang/:title"
}
]
}
This puts the value of the lang
parameter into the request parameters, making it available in req.params
when using Events. A URL such as /fr/about
would populate req.params
like this:
{
"lang": "fr"
}
Anchor link Query string
Using the query string, you can put the lang
parameter into the URL like so: /about?lang=fr
would again populates req.params
like this:
{
"lang": "fr"
}
Anchor link Unfiltered response
If you don't pass a language parameter to API, the response will contain the raw content of documents, containing the original value and all the language variations of each translatable field. In this case, no _i18n
field is added to the documents.
{
"_id": "58176e72bafa53b625aebd4f",
"title": "The Little Prince",
"title:pt": "O Principezinho",
"title:fr": "Le Petit Prince",
"author": "Antoine de Saint-Exupéry"
}
Anchor link Performance
Anchor link Caching
Caching is enabled by default, but disabled when ”debug”: true
. You can configure the cache headers for each MIME type, or disable the cache entirely in your config:
"caching" {
"directory": {
"enabled: false
}
}
Anchor link Compression
Web supports gzip
and br
(Brotli) compression and is on for all supported file-types by default. You can disable it globally:
"headers" {
"useCompression": {
"enabled: false
}
}
Anchor link Cache flush
DADI Web has a cache invalidation endpoint which allows an authorised user to flush the cache for either a specific path or the entire application. This process clears both page (HTML) and datasource (JSON) cache files.
The user must send a POST request to /api/flush
with a request body containing 1) the path to flush, and 2)
a set of credentials that match those held in the configuration file's auth
block:
Anchor link Flush all cache files
POST /api/flush HTTP/1.1
Host: www.example.com
{ "path": "*", "clientId": "your-client-id", "secret": "your-secret" }
Anchor link Flush cache files for a specific path
POST /api/flush HTTP/1.1
Host: www.example.com
{ "path": "/books/crime", "clientId": "testClient", "secret": "superSecret" }
Anchor link Serving static assets and content
Anchor link Public folder
The public folder is where you can store any static files you may need to serve to a browser (e.g., CSS, JavaScript, video files etc), the path can be configured to any location you like.
Content in this folder obeys your useCompression
and cacheControl
settings.
Anchor link Virtual directories
Virtual directories a similar to the public folder but are particularly geared to serving static content. You can list as many as you require in your config as an array:
"virtualDirectories": [
{
"path": "data/legacy_features",
"index": "default.html",
"forceTrailingSlash": false
}
]
index
is a similar function to a traditional web server where hitting the root of a folder will serve that document without a URI e.g., /legacy_features/
will serve /legacy_features/default.html
to the browser.
index
and forceTrailingSlash
are both optional.
Anchor link Debugging
Anchor link Log formatting
You can format the logs for easier readability by using Bunyan:
npm install -g bunyan
Then start the app:
npm start | bunyan -o short
Anchor link Debug view
When the config option is set to true
you can append ?debug
to any DADI Web URL and you will see how Web constructed that page.
This will look similar to the following:
From here you can see how to construct you templates to output specific variable or loop over particular objects. It is also useful for seeing the output of any Events you have which may output values into the page.
Anchor link Security
Anchor link CSRF tokens
Cross-Site Request Forgery (CSRF) is an attack that forces an end user to execute unwanted actions on a web application in which they're currently authenticated. CSRF attacks specifically target state-changing requests, not theft of data, since the attacker has no way to see the response to the forged request. More about CSRF.
CSRF protection allows developers to use a per-request CSRF token which will be injected into the view model, and ensures that all POST requests supply a correct CSRF token. Without a correct token, with CSRF enabled, users will be greeted with a 403.
To enable CSRF, set the security.csrf
config option in your config/config.{env}.json
file:
"security": {
"csrf": true
}
Once enabled, the variable csrfToken
will be injected into the viewModel. You will need to add this to any forms which perform a POST
using the field name _csrf
, like so:
<form action="/" method="post">
<input type="text" name="test_input_safe">
<input type="hidden" name="_csrf" value="{csrfToken}">
<input type="submit" value="Submit form">
</form>
If the CSRF token provided is incorrect, or one isn't provided, then a 403 forbidden
error will occur.
A working example can be found here: dadi-web-csrf-test.
Anchor link SSL
To use SSL you first need to generate the necessary certificates. This will vary depending on your platform. Digital Ocean has a useful introduction that may assist.
After that you need to tell Web where to find your SSL files and enable http
in your config.json
file.
"server": {
"host": "127.0.0.1",
"port": 443,
"protocol": "https",
"sslPassphrase": "superSecretPassphrase",
"sslPrivateKeyPath": "keys/server.key",
"sslCertificatePath": "keys/server.crt"
}
Anchor link Application status
DADI Web has an endpoint which returns a JSON object containing information about an application's platform, process and health state. The information returned by the endpoint includes:
- Latest version of the DADI Web application
- Node.js version
- Process ID
- Process uptime
- Process memory usage
- System host name
- System platform and version
- System uptime
- Memory (free and total)
- Current load averages
Anchor link Health Check Routes
In addition to system information, if any routes are specified in the configuration Web will send a request to each one and return data about the response. Each route is a JSON object containing the URL to hit and the expected response time in seconds:
{
"route": "/movies/latest",
"expectedResponseTime": 1
}
The response will contain a block for each of the routes configured, along with the HTTP status that was received, the response time, and a colour value indicating how the response performed against the configured response time.
"routes": [
{
"route": "/movies/latest",
"status": 200,
"responseTime": 0.039,
"healthStatus": "Green"
}
]
Anchor link Configuration
To enable the status endpoint, add a configuration block:
{
"status": {
"routes": [
{
"route": "/movies/latest",
"expectedResponseTime": 1
}
]
}
}
Anchor link User-Agent identifier
The following User-Agent header is used when making health check requests: 'User-Agent': '@dadi/status'
Anchor link Status request
Send a POST request to /api/status
with a request body containing a set of credentials that match those held in the configuration file's auth
block:
POST /api/status HTTP/1.1
Host: www.example.com
Content-Type: application/json
{ "clientId": "your-client-id", "secret": "your-secret" }
Anchor link Status response
With the configuration given above, expect a response similar to the following:
{
"service": {
"site": "Your Web Application",
"package": "@dadi/web",
"versions": {
"current": "6.0.0",
"latest": "6.0.1"
}
},
"process": {
"pid": 19463,
"uptime": 3.523,
"uptimeFormatted": "0 days 0 hours 0 minutes 3 seconds",
"versions": {
"http_parser": "2.3",
"node": "0.12.0",
"v8": "3.28.73",
"uv": "1.0.2",
"zlib": "1.2.8",
"modules": "14",
"openssl": "1.0.1l"
}
},
"memory": {
"rss": "86.508 MB",
"heapTotal": "65.771 MB",
"heapUsed": "32.938 MB"
},
"system": {
"platform": "darwin",
"release": "14.5.0",
"hostname": "hudson",
"memory": {
"free": "37.781 MB",
"total": "8.000 GB"
},
"load": [
2.2958984375,
2.27197265625,
2.25927734375
],
"uptime": 155084,
"uptimeFormatted": "1 days 19 hours 4 minutes 44 seconds"
},
"routes": [
{
"route": "/movies/latest",
"responseTime": 2,
"healthStatus": "Amber"
}
]
}
Anchor link How-to guides
Anchor link Migrating from version 2.x to 3.x
1. Install Dust.js dependency
Web 3.0 supports multiple template engines. As a consequence, Dust.js is now decoupled from core and needs to be included as a dependency on projects that want to use it.
npm install @dadi/web-dustjs --save
2. Change bootstrap script
The bootstrap script (which you may be calling index.js
, main.js
or server.js
) now needs to inform Web of the engines it has available and which npm modules implement them.
require('@dadi/web')({
engines: [
require('@dadi/web-dustjs')
]
})
3. Update config
The dust
config block has been moved inside a generic engines
block.
Before:
"dust": {
"cache": true,
"debug": true,
"debugLevel": "DEBUG",
"whitespace": true,
"paths": {
"helpers": "workspace/utils/helpers"
}
}
After:
"engines": {
"dust": {
"cache": true,
"debug": true,
"debugLevel": "DEBUG",
"whitespace": true,
"paths": {
"helpers": "workspace/utils/helpers"
}
}
}
4. Move partials directory
Before Web 3.0, Dust templates were separated between the pages
and partials
directories, with the former being used for templates that generate a page (i.e. have a route) and the latter being used for partials/includes.
In Web 3.0, all templates live under the same directory (pages
). The distinction between a page and a partial is made purely by whether or not the template has an accompanying JSON schema file.
Also, pages and partials can now be located in sub-directories, nested as deeply as possible.
To migrate an existing project, all you need to do is move the partials
directory inside pages
and everything will work as expected.
Before:
workspace
|_ pages
|_ partials
After:
workspace
|_ pages
|_ partials
mv workspace/partials workspace/pages
5. Update Dust helpers
If your project is using custom helpers, you might need to change the way they access the Dust engine. You should now access the module directly, rather than reference the one from Web.
// Before
var dust = require('@dadi/web').Dust
require('@dadi/dustjs-helpers')(dust.getEngine())
// After
var dust = require('dustjs-linkedin')
require('@dadi/dustjs-helpers')(dust)
Anchor link Dealing with form data and SendGrid
This is an example of an Event which uses SendGrid to send a message from an HTML form.
workspace/pages/contact.dust
{?mailResult}<p>{mailResult}</p>{/mailResult}
<form action="/contact/" method="post">
<p>
<label class="hdr" for="name">Name</label>
<input autofocus id="name" name="name" placeholder="Your full name" class="normal" type="text">
</p>
<p>
<label class="hdr" for="email">Email</label>
<input id="email" name="email" required placeholder="Your email address" class="normal" type="email">
</p>
<p>
<label class="hdr" for="phone">Phone</label>
<input id="phone" name="phone" placeholder="Contact telephone number" class="normal" type="text">
</p>
<p>
<label class="hdr" for="message">Message</label>
<textarea style="min-height:166px" rows="5" id="message" name="message" required placeholder="What do you want to talk about?" class="normal" type="email"></textarea>
</p>
<p>
<button type="submit">Send message</button>
</p>
</form>
You need an API key from SendGrid to use this Event in your application. Once you have obtained an API key from SendGrid.com, DADI Web should be started with an environment variable containing the API key. You will also need to whitelist your IP address within the SendGrid dashboard.
You could hardcode your API key, but be careful not to commit the code to a publicly accessible GitHub repo.
Starting DADI Web with an environment variable
$ SENDGRID_API=71713987-9f01-4dea-b3d4-8d0bcd9d53ed node index.js
workspace/events/contact.js
const sendgrid = require('sendgrid')(process.env['SENDGRID_API'])
const Event = function (req, res, data, callback) {
// On form post
switch (req.method.toLowerCase()) {
case 'post':
// Validate the inputs
if (req.body.email && isEmail(req.body.email) && req.body.message) {
const message = "Name: " +req.body.name + "\n\nEmail: " + req.body.email + "\n\nPhone: " + req.body.phone + "\n\nMessage:\n\n" + req.body.message
var request = sendgrid.emptyRequest({
method: 'POST',
path: '/v3/mail/send',
body: {
personalizations: [{
to: [{
email: 'hello@dadi.cloud',
}],
subject: '[dadi.cloud] Contact form message',
}],
from: {
email: 'hello@dadi.cloud',
},
content: [{
type: 'text/plain',
value: message,
}]
}
})
sendgrid.API(request, (error, response) => {
if (error) {
data.mailResult = 'There was a problem sending the email.'
} else {
data.mailResult = 'Thank you for your message, you will hear back from us soon.'
}
callback()
})
} else {
data.mailResult = 'All fields are required.'
callback()
}
break
default:
return callback()
}
}
// Taken from: http://stackoverflow.com/a/46181/306059
function isEmail(email) {
const re = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/
return re.test(email)
}
module.exports = function (req, res, data, callback) {
return new Event(req, res, data, callback)
}