Anchor link Installation

Anchor link Requirements

Microservices in the DADI platform are built on Node.js, a JavaScript runtime built on Google Chrome's V8 JavaScript engine. Node.js uses an event-driven, non-blocking I/O model that makes it lightweight and efficient.

DADI follows the Node.js LTS (Long Term Support) release schedule, and as such the version of Node.js required to run DADI products is coupled to the version of Node.js currently in Active LTS. See the LTS schedule for further information.

• Node.js (current LTS version)

Anchor link DADI CLI

The easiest way to install Web is using DADI CLI. CLI is a command line application that can be used to create and maintain installations of DADI products.

Anchor link Install DADI CLI

$ npm install @dadi/cli -g

Anchor link Create new Web installation

There are two ways to create a new Web application with the CLI: either manually create a new directory for Web or let CLI handle that for you. DADI CLI accepts an argument for project-name which it uses to create a directory for installation.

Manual directory creation

$ mkdir my-web-app
$ cd my-web-app
$ dadi web new

Automatic directory creation

$ dadi web new my-web-app
$ cd my-web-app

Anchor link NPM

All DADI platform microservices are available from NPM. To add Web to your existing project as a dependency:

$ cd my-existing-node-app
$ npm install @dadi/web

This will create config & workspace folders and server.js which will serve as the entry point to your app.

Anchor link Application Anatomy

When CLI finishes creating your Web instance, the application directory will contain the basic requirements for launching your Web instance. The following directories and files have been created for you:

my-web/
  config/              # contains environment-specific configuration files
    config.development.json
  server.js            # the entry point for the application
  package.json
  workspace/
    datasources/       # datasource specification files
    events/            # event files - files run before page render
    pages/             # page template and specification (.json) files
      partials/        # page template includes
    posts/             # markdown files as a blog example
    public/            # files to expose raw to the web, e.g. icons, images, styles, scripts
    utils/
      helpers/         # globally accessible functions that templates can call
    collections/       # collection specification files
    endpoints/         # custom JavaScript endpoints

Anchor link Configuration

All the core platform services are configured using environment specific configuration.json files, the default being development. For more advanced users this can also load based on the hostname i.e., it will also look for config." + req.headers.host + ".json

The minimal config.development.json file looks like this:

{
  "server": {
    "host": "localhost",
    "port": 3000
  },
  "cluster": false
}

Anchor link Advanced configuration

Anchor link Example Configuration File

{
    "app": {
      "name": "Project Name Here"
    },
    "server": {
      "host": "127.0.0.1",
      "port": 443,
      "socketTimeoutSec": 30,
      "protocol": "https",
      "redirectPort": 80,
      "sslPassphrase": "superSecretPassphrase",
      "sslPrivateKeyPath": "keys/server.key",    
      "sslCertificatePath": "keys/server.crt"
    },      
    "api": {
      "host": "127.0.0.1",
      "port": 3000
    },
    "auth": {
      "tokenUrl":"/token",
      "clientId":"webClient",
      "secret":"secretSquirrel"
    },
    "aws": {
      "accessKeyId": "<your key here>",
      "secretAccessKey": "<your secret here>",
      "region": "eu-west-1"
    },
    "caching": {
      "ttl": 300,
      "directory": {
        "enabled": true,
        "path": "./cache/web/",
        "extension": "html"
      },
      "redis": {
        "enabled": false,
        "host": "localhost",
        "port": 6379
      }
    },
    "engines": {
      "dust": {
        "cache": true,
        "debug": true,
        "debugLevel": "DEBUG",
        "whitespace": true,
        "paths": {
          "helpers": "workspace/utils/helpers"
        }
      }
    },
    "headers": {
      "useCompression": true,
      "cacheControl": {
        "text/css": "public, max-age=86400"
      }
    },
    "logging": {
      "enabled": true,
      "level": "info",
      "path": "./log",
      "filename": "dadi-web",
      "extension": "log",
      "accessLog": {
        "enabled": true,
        "kinesisStream": "dadi_web_test_stream"
      }
    },
    "paths": {
      "datasources": "./workspace/datasources",
      "events": "./workspace/events",
      "middleware": "./workspace/middleware",
      "pages": "./workspace/pages",
      "partials": "./workspace/partials",
      "public": "./workspace/public",
      "routes": "./workspace/routes",
      "helpers": "./workspace/utils/helpers",
      "filters": "./workspace/utils/filters"
    },
    "rewrites": {
      "datasource": "redirects",
      "path": "workspace/routes/rewrites.txt",
      "forceLowerCase": true,
      "forceTrailingSlash": true,
      "stripIndexPages": ['index.php', 'default.aspx']
    },
    "global" : {
      "baseUrl": "http://www.example.com"
    },
    "globalEvents": [
      "timestamp"
    ],
    "debug": true,
    "allowDebugView": true
}

You can see all the config options in config.js.

Anchor link app

Property	Type	Default	Description	Example
name	String	DADI Web (Repo Default)	The name of your application, used for the boot message	My project

Anchor link server

Property	Type	Default	Description	Example
host	String	0.0.0.0	The hostname or IP address to use when starting the Web server	example.com
port	Number	8080	The port to bind to when starting the Web server	80
socketTimeoutSec	Number	30	The number of seconds to wait before closing an idle socket	10
protocol	String	http	The protocol the web application will use	https
redirectPort	Number	-	A port to redirect from, to the port	80
sslPassphrase	String	-	The passphrase of the SSL private key	secretPassword
sslPrivateKeyPath	String	-	The path to the SSL private key	/etc/ssl/key.pem
sslCertificatePath	String	-	The filename of the SSL certificate	/etc/ssl/cert.pem
sslIntermediateCertificatePath	String	-	The filename of an SSL intermediate certificate, if any	/etc/ssl/ca.pem
sslIntermediateCertificatePaths	Array	-	The filenames of SSL intermediate certificates, overrides sslIntermediateCertificatePath (singular)	[ '/etc/ssl/ca/example.pem', '/etc/ssl/ca/other.pem' ]

Anchor link api

Property	Type	Default	Description	Example
host	String	0.0.0.0	The hostname or IP address of the DADI API instance to connect to	api.example.com
protocol	String	http	The protocol to use	https
port	Number	8080	The port of the API instance to connect to	3001

Alternatively you can specify an object of configuration objects if you intend to use multiple data provider configurations. For example:

"api": { 
    "main": {
        "host": "127.0.0.1",
        "port": 3000,
        "auth": {
            "tokenUrl": "/token",
            "clientId": "your-client-id",
            "secret": "your-secret"
        }
    },
    "secondary":  {
        "host": "127.0.0.1",
        "port": 3000,
        "auth": {
            "tokenUrl": "/token",
            "clientId": "your-client-id",
            "secret": "your-secret"
        }
    }
}

You can then reference the api config in your datasource specification:

{
  "datasource": {
      "key": "articles",
      "source": {
          "api": "main",
          "endpoint": "1.0/cloud/articles"
      },
      "count": 12,
      "paginate": false
}

The defaults order for finding API information is:

• The api and auth blocks
• Or an api configuration defined with "type": "dadiapi"
• Or an api with no defined type
• Or the api in position api[0]
• Or settings in the source of the datasource itself

Anchor link auth

This block is used in conjunction with the api block above, but is also used in calls to the status and cache flush endpoints.

Property	Type	Default	Description	Example
tokenUrl	String	/token	The endpoint to use when requesting Bearer tokens from DADI API	anotherapi.example.com/token
protocol	String	http	The protocol to use when connecting to the tokenUrl	https
clientId	String	your-client-key	Should reflect what you used when you setup your DADI API	my-user
secret	String	your-client-secret	The corresponding password	my-secret

Anchor link caching

N.B. Caching across DADI products is standardised by DADI Cache.

Property	Type	Default	Description	Example
ttl	Number	300	The time, in seconds, after which cached data is considered stale	3600

Anchor link directory

Property	Type	Default	Description	Example
enabled	Boolean	true	If enabled, cache files will be saved to the filesystem	false
path	String	./cache/web	Where to store the cache files	./tmp
extension	String	html	The default file extension for cache files. Note that Web will override this if compression is enabled	json

Anchor link redis

You will need to have a Redis server running to use this.

Property	Type	Default	Description	Example
enabled	Boolean	true	If enabled, cache files will be saved to specified Redis server	false
cluster	Boolean	false
host	String	127.0.0.1
port	Number	6379
password	String	-

Anchor link engines

In version 3.0 and above, DADI Web can handle multiple template engines, the default being a Dust.js interface. You can pass configuration options to these adaptors in this block.

Please see Views later for more information.

Anchor link headers

Property	Type	Default	Description	Example
useGzipCompression (deprecated, see useCompression)	Boolean
useCompression	Boolean	true	Attempts to compress the response, including assets, using either Brotli or Gzip	false
cacheControl	Object	{ 'image/png': 'public, max-age=86400', 'image/jpeg': 'public, max-age=86400', 'text/css': 'public, max-age=86400', 'text/javascript': 'public, max-age=86400', 'application/javascript': 'public, max-age=86400', 'image/x-icon': 'public, max-age=31536000000' }	A set of custom cache-control headers (in seconds) for different content types

In addition, a cacheControl header can be used for a 301/302 redirect by adding to the configuration block:

"headers": {
  "cacheControl": {
    "301": "no-cache"
  }
}

Anchor link logging

Property	Type	Default	Description	Example
enabled	Boolean	true	If true, logging is enabled using the following settings.	false
level	debug,info,warn,error,trace	info	The level at which log messages will be written to the log file.	warn
path	String	./log	The absolute or relative path to the directory for log files	/data/app/log
filename	String	web	The filename to use for the log files. The name you choose will be given a suffix indicating the current application environment	my_application_name
extension	String	log	The extension to use for the log files	txt

Anchor link accessLog

Property	Type	Default	Description	Example
enabled	Boolean	true	If true, HTTP access logging is enabled. The log file name is similar to the setting used for normal logging, with the addition of 'access'. For example dadi-web.access.log	false
kinesisStream	String	-	An AWS Kinesis stream to write to log records to	web_aws_kinesis

Anchor link aws

For use with the above block logging.accessLog.

Property	Type	Default	Description	Example
accessKeyId	String
secretAccessKey	String
region	String

Anchor link rewrites

Property	Type	Default	Description	Example
datasource	String	-	The name of a datasource used to query the database for redirect records matching the current URL. More info	redirects
path	String	-	The path to a file containing rewrite rules	workspace/routes/rewrites.txt
forceLowerCase	Boolean	false	If true, converts URLs to lowercase before redirecting	true
forceTrailingSlash	Boolean	false	If true, adds a trailing slash to URLs before redirecting	true
stripIndexPages	Array	-	A set of common index page filenames to remove from URLs	[‘index.php', 'default.aspx']

Anchor link global

The global section can be used for any application parameters that should be available for use in page templates, such as asset locations, 3rd party account identifiers, etc

"global" : {
  "baseUrl": "http://www.example.com"
}

In the above example baseUrl would be available to a page template and could be used in the following way:

<html>
<body>
  <h1>Welcome to DADI Web</h1>
  <img src="{global.baseUrl}/images/welcome.png"/>
</body>
</html>

Anchor link globalEvents

Events to be loaded on every request.

Anchor link paths

Paths can be used to configure where any folder of the app assets are located.

For example:

"paths": {
  "workspace": "workspace",
  "datasources": "workspace/datasources",
  "pages": "workspace/pages",
  "events": "workspace/events",
  "middleware": "workspace/middleware",
  "media": "workspace/media",
  "public": "workspace/public",
  "routes": "workspace/routes"
}

Anchor link debug

See debugging

If set to true, Web logs more information about routing, caching etc. Caching is also disabled.

Anchor link allowDebugView

See Debug view

If set, enabled the page debug view to be accessible by appending the querystring ?debug to the end of any URL.

Anchor link Environmental variables

Best practice is to avoid keeping sensitive information inside a config.*.json. Therefore anywhere a password or secret is used in a config file can be substituted for an environmental variable.

Variable	Block to substitute
AUTH_TOKEN_ID	auth.clientId
AUTH_TOKEN_SECRET	auth.secret
AWS_ACCESS_KEY	aws.accessKeyId
AWS_SECRET_KEY	aws.secretAccessKey
AWS_REGION	aws.region
NODE_ENV	env
REDIS_HOST	caching.redis.host
REDIS_PORT	caching.redis.post
REDIS_PASSWORD	caching.redis.password
SESSION_SECRET	sessions.secret
PORT	server.port
PROTOCOL	server.potocol
SSL_PRIVATE_KEY_PASSPHRASE	server.sslPassphrase
SSL_PRIVATE_KEY_PATH	server.sslPrivateKeyPath
SSL_CERTIFICATE_PATH	server.sslCertificatePath
SSL_INTERMEDIATE_CERTIFICATE_PATH	server.sslIntermediateCertificatePath
SSL_INTERMEDIATE_CERTIFICATE_PATHS	server.sslIntermediateCertificatePaths
TWITTER_CONSUMER_KEY	twitter.consumerKey
TWITTER_CONSUMER_SECRET	twitter.consumerSecret
TWITTER_ACCESS_TOKEN_KEY	twitter.accessTokenKey
TWITTER_ACCESS_TOKEN_SECRET	twitter.accessTokenSecret
WORDPRESS_BEARER_TOKEN	workspress.bearerToken

Anchor link Adding pages

A page on your website consists of two files within your workspace: a JSON specification and a template.

N.B. The location of this folder is configurable, but defaults to workspace/pages.

Anchor link Example specification

Here is an example page specification, with all options specified.

{
  "page": {
    "name": "People",
    "description": "A page for displaying People records."
  },
  "settings": {
    "cache": true,
    "beautify": true,
    "keepWhitespace": true,
    "passFilters": true
  },
  "routes": [
    {
      "path": "/people"
    }
  ],
  "contentType": "text/html",
  "template": "people.dust",
  "datasources": [
    "allPeople"
  ],
  "requiredDatasources": [
    "allPeople"
  ],
  "events": [
    "processPeopleData"
  ],
  "preloadEvents": [
    "geolocate"
  ]
}

Anchor link page

Property	Type	Default	Description
name	String	-	Used by the application for identifying the page internally.

Any other properties you add are passed to the page data. Useful for maintaining HTML <meta> tags, languages etc.

Anchor link settings

Property	Type	Default	Description
cache	Boolean	Reflects the caching settings in the main config file	Used by the application for identifying the page internally.

Anchor link routes

For every page added to your application, a route is created by default. A page’s default route is a value matching the page name. For example if the page name is books the page will be available in the browser at /books.

To make the books page reachable via a different URL, simply add (or modify) the page’s routes property:

"routes": [
  {
    "path": "/reading"
  }
]

For detailed documentation of routing, see Routing.

Anchor link Content Type

The default content type is text/html. This can be overridden by defining the contentType in the root of the page config.

"contentType": "application/xhtml+xml"

Anchor link Templates

Template files are stored in the same folder as the page specifications and by default share the same filename as the page.json. Unless the page specification contains an explicit template property, the template name should match the page specification name.

See Views for further documentation.

Anchor link Datasources

An array containing datasources that should be executed to load data for the page.

For detailed documentation of datasources, see Datasources

"datasources": [
  "datasource-one",
  …
]

Anchor link Required Datasources

Allows specifying an array of datasources that must return data for the page to function. If any of the listed datasources return no results, a 404 is returned. The datasources specified must exist in the datasources array.

"requiredDatasources": [
  "datasource-one",
  ...
]

Anchor link Events

An array containing events that should be executed after the page's datasources have loaded data.

"events": [
  "event-one",
  ...
]

For detailed documentation of events, see Events

Anchor link Preload Events

An array containing events that should be executed before the rest of the page's datasources and events.

Preload events are loaded from the filesystem in the same way as a page's regular events, and a Javascript file with the same name must exist in the events path.

"preloadEvents": [
  "preloadevent-one",
  ...
]

Anchor link Caching

If true the output of the page will be cached using cache settings in the main configuration file.

"settings": {
  "cache": true
}

For detailed documentation of page caching, see Caching.

Anchor link Routing, rewrites and redirects

Routing allows you to define URL endpoints for your application and control how Web responds to client requests.

Anchor link Basic Routing

Adding routes provides URLs for interacting with the application. A route specified as /contact-us, for example, will make a URL available to your end users as http://www.example.com/contact-us.

Anchor link Page Routing

For every page added to your application, a route is created by default. A page's default route is a value matching the page name. For example if the page name is books the page will be available in the browser at /books.

To make the books page reachable via a different URL, simply add (or modify) the page's routes property:

{
  "routes": [
    {
      "path": "/reading"
    }
  ]
}

Anchor link Dynamic parameters

Routes may contain dynamic segments or named parameters which are resolved from the request URL and can be utilised by the datasources and events attached to the page.

A route segment with a colon at the beginning indicates a dynamic segment which will match any value. For example, a page with the route /books/:title will be loaded for any request matching the format. DADI Web will extract the :title parameter and add it to the req.params object, making it available for use in the page's attached datasources and events.

The following URLs match the above route, with the segment defined by :title extracted, placed into req.params and accessible via the property title.

URL	Named Parameter :title	Request Parameters req.params
/books/war-and-peace	war-and-peace	{ title: "war-and-peace" }
/books/sisters-brothers	sisters-brothers	{ title: "sisters-brothers" }

Anchor link Optional Parameters

Parameters can be made optional by adding a question mark ?.

For example the route /books/:page? will match requests in both the following formats:

	URL	Matched?	Named Parameters	Request Parameters req.params
	/books	Yes		{}
	/books/2	Yes	:page	{ page: "2" }

Anchor link Parameter Format

Specifying a format for a parameter can help Web identify the correct route to use. We can use the same example as above, where the URL has an optional page parameter. If we add a regular expression to this parameter indicating that it should only match numbers, any URL that doesn't contain numbers in this segment will not match the route.

Example

The route /books/:page(\\d+) will only match a URL that has books in the first segment and a number in the second segment:

	URL	Matched?	Named Parameters	Request Parameters req.params
	/books/war-and-peace	No
	/books/2	Yes	:page	{ page: "2" }

N.B. DADI Web uses the Path to Regexp library when parsing routes and parameters. More information on parameter usage can be found in the Github repository.

Anchor link Multiple Route Pages

The routes property makes it easy for you to define "multiple route" pages, where one page specification can handle requests for multiple routes.

DADI Web versions >= 1.7.0

DADI Web 1.7.0 introduced a more explicit way of specifying multiple routes per page . The route property has been replaced with routes which should be an Array of route objects.

Each route object must contain, at the very least, a path property. At startup, Web adds the value of each path property to an internal collection of routes for matching incoming requests.

{
  "routes": [
    {
      "path": "/movies/:title"
    },
    {
      "path": "/movies/news/:title?/"
    },
    {
      "path": "/movies/news/:page?/"
    }
  ]
}

In the above example, the same page (and therefore it's template) will be loaded for requests matching any of the formats specified by the path properties:

http://web.somedomain.tech/movies/deadpool
http://web.somedomain.tech/movies/news/
http://web.somedomain.tech/movies/news/2
http://web.somedomain.tech/movies/news/deadpool

Anchor link Route Priority

DADI Web sorts your routes into a priority order so that the most likely matches are easier to find.

• In Web, the most important parts of a route are the static segments, or rather the non-dynamic segments, for example /books. The more static segments in a route the higher its priority.
• The second most important parts are the mandatory dynamic segments, for example /:title.
• The least important parts are the optional dynamic segments, for example /:year?.
• Any route with a page parameter gets a slight edge, with 1 point being added to its priority.

Path	Priority
/movies/news/:page(\\d+)?/	12
/movies/reviews/:page(\\d+)?	12
/movies/features/:page(\\d+)?/	12
/movies/news/:title?/	11
/movies/features/:title?/	11
/movies/reviews/	10
/movies/:title/:page(\\d+)?	9
/movies/:title/:content?	8
/movies/	5

Anchor link Route Validation

An application may have more than one route that matches a particular URL, for example two routes that each have one dynamic segment:

/:genres
/:categories

In this case it is possible to provide DADI Web with some rules for determining the correct routes based on the parameters in the request. Parameter checks currently supported are:

• preload - tests the parameter value exists in a set of preloaded data
• in - tests the parameter value exists in an array of static values
• fetch - performs a datasource lookup using the parameter value as a filter

Anchor link Parameter Validation

Anchor link Preloaded data (preload)

To validate parameters against preloaded data, you first need to configure Web to preload some data. Add a block to the main configuration file like the example below, using your datasource names in place of "channels":

{
  "data": {
    "preload": [
      "channels"
    ]
  }
}

{
  "routes": [
    {
      "path": "/:channel/news/",
      "params": [
        {
          "param": "channel",
          "preload": {
            "source": "channels",
            "field": "key"
          }
        }
      ]
    }
  ]
}

Anchor link Static array test (in)

{
  "routes": [
    {
      "path": "/movies/:title/:subPage?/",
      "params": [
        {
          "param": "subPage",
          "in": ["review"]
        }
      ]
    }
  ]
}

Anchor link Datasource lookup (fetch)

{
  "routes": [
    {
      "path": "/movies/:title/:content?/",
      "params": [
        {
          "fetch": "movies"
        }
      ]
    }
  ]
}

Anchor link Route Constraint Functions

In the case of ambiguous routes it is possible to provide DADI Web with a constraint function to check each matching route against some business logic or existing data.

Returning true from a constraint instructs DADI Web that this is the correct route, the attached datasources and events should be run and the page displayed.

Returning false from a constraint instructs DADI Web to try the next matching route (or return a 404 if there are no further matching routes).

Constraints are added as a route property in the page specification file:

{
  "routes": [
    {
      "path": "/:people",
      "constraint": "nextIfNotPeople"
    }
  ]
}

To add constraint functions, create a file in the routes folder (by default configured as app/routes). The file MUST be named constraints.js.

In the following example the route has a dynamic parameter subPage. The constraint function nextIfNewsOrFeatures will check the value of the subPage parameter and return false if it matches "news" or "features", indicating to DADI Web that the next matching route should be tried (or a 404 returned if there are no further matching routes).

app/pages/movies.json

{
  "routes": [
    {
      "path": "/movies/:subPage",
      "constraint": "nextIfNewsOrFeatures"
    }
  ]
}

app/routes/constraints.js

module.exports.nextIfNewsOrFeatures = function (req, res, callback) {  
  if (req.params.subPage === 'news' || req.params.subPage === 'features') {
    return callback(false)
  }
  else {
    return callback(true)
  }
}
}

Anchor link Constraint Datasources

Note: Deprecated in Version 1.7.0

An existing datasource can be used as the route constraint. The specified datasource must exist in datasources (by default configured as app/datasources). The following examples have some missing properties for brevity.

app/pages/books.json

{
  "route": {
    "paths": ["/:genre"],
    "constraint": "genres"
  }
}

app/datasources/genres.json

{
  "datasource": {
    "key": "genres",
    "name": "Genre datasource",
    "source": {
      "endpoint": "1.0/library/genres"
    },
    "count": 1,
    "fields": { "name": 1, "_id": 0 },
    "requestParams": [
      { "param": "genre", "field": "title" }
    ]
  }
}

In the above example a request for http://www.example.com/crime will call the genres datasource, using the requestParams to supply a filter to the endpoint. The request parameter :genre will be set to crime and the resulting datasource endpoint will become:

/1.0/library/genres?filter={"title":"crime"}

If there is a result for this datasource query, the constraint will return true, otherwise false.

Anchor link Template URL Building

Using toPath():

var app = require('dadi-web');
var page = app.getComponent('people');
var url = page.toPath({ id: '1234' });

"/person/1234"

Anchor link Using Request Parameters

See Datasource Specification for more information regarding the use of named parameters in datasource queries.

Anchor link URL Rewriting and Redirects

Anchor link forceLowerCase

With this property set to true, Web converts incoming URLs to lowercase and sends a 301 Redirect to the browser with the lowercased version of the URL.

{
  "forceLowerCase": true
}

Anchor link forceTrailingSlash

With this property set to true, Web adds a trailing slash to incoming URLs and sends a 301 Redirect to the browser with the new version of the URL.

{
  "forceTrailingSlash": true
}

Anchor link stripIndexPages

This property accepts an array of filenames to remove from URLs. Useful for when you're migrating from another system and search engines may have indexed URLs containing legacy files. For example http://legacy-web.example.com/index.php

{
  "stripIndexPages": ["index.php", "default.aspx"]
}

Anchor link URL Rewriting

URL rewriting support in DADI Web is similar to Apache's mod_rewrite module. To setup, add your rewrite rules to a text file, one rule per line, and update the Web configuration with the path to your file:

main configuration file

"rewrites": {
  "path": "workspace/routes/rewrites.txt"
}

Anchor link Rewrite Rules

Each rewrite rule consists of three sections: a match, a replacement and a set of flags. The "match" section allows for regular expression matching of request URLs. If a match is found, the request is modified to use the contents of the "replacement" section. The "flags" modify the behaviour of the match or the final request.

Anchor link Defined parameters

Defined parameters can be wrapped with () and then accessed in the replacement with $n. For example, if migrating to a new URL structure that no longer has the "blog/" portion, it is possible to redirect all legacy URLs to the new structure using the following rule. The only defined parameter is "everything" following the "blog/" part of the URL, and we're asking Web to use only the contents of that parameter as the replacement:

^/blog/(.*)$ /$1 [L]

Will redirect /blog/2017/06/10/xyz to /2017/06/10/xyz

^/blog/(.*)$ /$1 [L]

Examples

1) Send a HTTP 301 redirect from /books/hemingway to /books?author=hemingway

^/books/(.*)$ /books?author=$1 [R=301,L]

2) Add a trailing slash to any URL if it doesn't have one already, sending a HTTP 301 redirect with the new URL

^(.*[^/])$ $1/ [R=301,L]

Anchor link Flags

Flags modify the behaviour of the URL rewriting system. Put the required flags within [] and separate them with commas.

• Last [L]: if a path matches, any subsequent rewrite rules will be disregarded
• Proxy [P]: proxy your requests ^/test/proxy/(.*)$ http://nodejs.org/$1 [P]
• Redirect [R], [R=301]`: issue a redirect for the request
• Nocase [NC]: regex match will be case-insensitive
• Forbidden [F]: issue a HTTP 403 Forbidden response
• Gone [G]: issue a HTTP 410 Gone response
• Type [T=*]: sets the content-type to the specified one (replace * with mime-type)
• Host [H], [H=*]: matches on the request host header (replace * with a regular expression that matches a hostname)

For more info about available flags, please see the Apache page.

Anchor link Redirecting domains

To redirect from one domain to another, for example to remove www. from requests and redirect them to the root domain, add a rule similar to the following. The Host flag is used, specifying the request host header that this rule should match. The "match" section specifies a single defined parameter which is used in the "replacement" section, appended to the hardcoded new domain.

\/(.*)$ https://dadi.tech/$1 [H=www\.dadi\.tech, R=302,NC,L]

Anchor link Views

Anchor link Engines

You can use a variety of template engines with DADI Web. We maintain several template engines, such as Dust, Pug and Handlebars. You can find more on NPM.

Anchor link Install a new engine

Each package lists it's own install instructions, but they all follow the same pattern:

Install the interface you want:

npm install @dadi/web-handlebars --save

Edit your app entry file (by default this is server.js):

require('@dadi/web')({
  engines: [
    require('@dadi/web-handlebars')
  ]
})

Anchor link Creating your own engine

Full instructions for this are available in our Web sample engine repo.

Anchor link Error pages

DADI Web has default error pages, but it will look for templates in the pages folder which match the error code needing to be expressed e.g., 404.dust.

Anchor link Sessions

DADI Web uses the express-session library to handle sessions. Visit that project's homepage for more detailed information regarding session configuration.

• Configuration
• Configuration Properties
• Using the session

Anchor link Session Configuration

Note: Sessions are disabled by default. To enable them in your application, add the following to your configuration file:

"sessions": {
  "enabled": true
}

A full configuration block for sessions contains the following properties:

"sessions": {
  "enabled": true,
  "name": "your-cookie-name"
  "secret": "your-secret-key",
  "resave": false,
  "saveUninitialized": false,
  "store": "",
  "cookie": {
    "maxAge": 60000,
    "secure": false
  }
}

Anchor link Session Configuration Properties

Property	Description	Default
enabled	If true, sessions are enabled.	false
name	The session cookie name.	"dadiweb.sid"
secret	The secret used to sign the session ID cookie. This can be either a string for a single secret, or an array of multiple secrets. If an array of secrets is provided, only the first element will be used to sign the session ID cookie, while all the elements will be considered when verifying the signature in requests.	"dadiwebsecretsquirrel"
resave	Forces the session to be saved back to the session store, even if the session was never modified during the request.	false
saveUninitialized	Forces a session that is "uninitialized" to be saved to the store. A session is uninitialized when it is new but not modified. Choosing false is useful for implementing login sessions, reducing server storage usage, or complying with laws that require permission before setting a cookie. Choosing false will also help with race conditions where a client makes multiple parallel requests without a session.	false
store	The session store instance, defaults to a new MemoryStore instance.	The default is an empty string, which uses a new MemoryStore instance. To use MongoDB as the session store, specify a MongoDB connection string such as "mongodb://host:port/databaseName" or "mongodb://username:password@host:port/databaseName" if your database requires authentication. To use Redis, specify a Redis server's address and port number: "redis://127.0.0.1:6379".
cookie
cookie.maxAge	Set the cookie’s expiration as an interval of seconds in the future, relative to the time the browser received the cookie. null means no 'expires' parameter is set so the cookie becomes a browser-session cookie. When the user closes the browser the cookie (and session) will be removed.	60000
cookie.secure	HTTPS is necessary for secure cookies. If secure is true and you access your site over HTTP, the cookie will not be set.	false

Anchor link Using the session

Session data can easily be accessed from an event or custom middleware.

const Event = (req, res, data, callback) => {
  if (req.session) {
    req.session.someProperty = "some value"
    req.session.save(function (err) {
      // session saved
    })

    data.session_id = req.session.id
  }

  callback(null, data)
}

Anchor link Adding data

Anchor link Datasources

Datasources are used to connect to both internal and external data providers to load data for rendering pages.

my-web/
  app/
    datasources/      
      books.json      # a datasource specification
    events/           
    pages/            

Anchor link Datasource Specification File

{
  "datasource": {
    "key": "books",
    "name": "Books datasource",
    "source": {
      "type": "dadiapi",
      "endpoint": "1.0/library/books"
    },
    "paginate": true,
    "count": 5,
    "sort": { "name": 1 },
    "filter": {},
    "fields": {}
  }
}

Anchor link Datasource Configuration

Property		Description	Default value	Example
key		Short identifier of the datasource. This value is used in the page specification files to attach a datasource		"books"
name		This is the name of the datasource, commonly used as a description for developers		"Books"
paginate			true	true
count		Number of items to return from the endpoint per page. If set to '0' then all results will be returned, up to the limit specified in the collection schema settings block	20	5
sort		A JSON object with fields to order the result set by	{} // unsorted	{ "title": 1 } // sort by title ascending, { "title": -1 } // sort by title descending
filter		A JSON object containing a MongoDB query		{ "SaleDate" : { "$ne" : null} }
filterEvent		An event file to execute which will generate the filter to use for this datasource. The event must exist in the configured events path		"getBookFilter"
fields		Limits the fields to return in the result set		{ "title": 1, "author": 1 }
requestParams		An array of parameters the datasource can obtain from the querystring, config or the session. See Passing Parameters for more.		[ { "param": "author", "field": "author_id" } ]
source
	type	(optional) Determines whether the data is from a remote endpoint or local, static data	"remote"	"remote", "static"
	protocol	(optional) The protocol portion of an endpoint URI	"http"	"http", "https"
	host	(optional) The host portion of an endpoint URL	The main config value api.host	"api.somedomain.tech"
	port	(optional) The port portion of an endpoint URL	The main config value api.port	3001
	endpoint	The path to the endpoint which contains the data for this datasource		"/1.0/news/articles"
caching
	enabled	Sets caching enabled or disabled	false	true
	ttl
	directory	The directory to use for storing cache files, relative to the root of the application		"./cache"
	extension	The file extension to use for cache files		"json"
auth
	type			"bearer"
	host			"api.somedomain.tech"
	port			3000
	tokenUrl			"/token"
	credentials			{ "clientId": "your-client-key", "secret": "your-client-secret" }

Anchor link Passing parameters

requestParams is an array of parameters that can be used to generate a datasource filter or even modify a datasource's endpoint. Web can obtain parameters from the request querystring, the configuration or from the session (if one exists).

The syntax for request parameters in a datasource specification looks like the following:

"requestParams": [
  {
    "source": "config|session|request",
    "param": "param",
    "field": "field",
    "target": "filter|endpoint"
  }
]

Property	Description	Default
source	Where to look for the parameter specified by "param"	"request"
param	The parameter or property that contains the value to be used
field	The name of the property to add to a filter, or the endpoint placeholder to replace
target	Whether this parameter should be added to a datasource filter, or directly into the endpoint	"filter"

Anchor link Extract parameters from the request (default)

The simplest usecase for requestParams is in conjunction with dynamic parameters obtained from route properties in a page specification.

For example, given a collection books with the fields _id, title

With the page route /books/:title and the below datasource specification, Web will extract the :title parameter from the URL and use it to query the books collection using the field title.

With a request to http://www.somedomain.tech/books/sisters-brothers, the named parameter :title is sisters-brothers. A filter query is constructed for the datasource using this value.

The resulting query passed to the underlying datastore will be: { "title" : "sisters-brothers" }

See Routing for detailed routing documentation.

Page specification

"routes": [{
  "path": "/books/:title"
}]

Datasource specification

"source": {
  "endpoint": "1.0/library/books"
},
"requestParams": [
  {
    "param": "title",
    "field": "title"
  }
]

In this case, the requestParam's param property is being used as the field to filter on and so must match a named parameter in the page specification's routes.

Anchor link Extract parameters from config

Values held in the configuration file can be used to replace placeholders in an endpoint. In the configuration below, a global section has been added, containing a news item to retrieve from Hacker News.

config/config.development.json

{
  "server": {
    "host": "0.0.0.0",
    "port": 3001
  },
  "global": {
    "newsItems": {
      "hackernewsItemId": "17200415"
    }
  }
}

With a Hacker News datasource, we can add a placeholder to the HN API endpoint {item} and have Web replace that with the value obtained from the configuration file:

{
  "datasource": {
    "key": "hackernews",
    "name": "Get specific news item",
    "source": {
      "type": "remote",
      "protocol": "https",
      "host": "hacker-news.firebaseio.com",
      "port": 443,
      "endpoint": "v0/item/{item}.json"
    },
    "requestParams": [
      {
        "source": "config",
        "param": "global.newsItems.hackernewsItemId",
        "field": "item",
        "target": "endpoint"
      }
    ]
  }
}

If you've forgotten to add the configuration property, Web will throw the following error:

[2018-06-01 13:14:43.444] [LOG]   Error: cannot find configuration param 'global.newsItems.hackernewsItemId'
✖ Error: cannot find configuration param 'global.newsItems.hackernewsItemId'

If configured successfully, Web will generate a request for the URL https://hacker-news.firebaseio.com:443/v0/item/17200415.json:

03:18:39.020Z  INFO dadi-web: GOT datasource "hackernews": https://hacker-news.firebaseio.com:443/v0/item/17200415.json (HTTP 200, 223 Bytes) (module=remote)

The resulting response will be added to the data context using the datasource's key (in this case hackernews):

"hackernews": {
  "by": "captn3m0",
  "descendants": 1,
  "id": 17200415,
  "kids": [
    17200762
  ],
  "score": 4,
  "time": 1527801802,
  "title": "Show HN: OPML generator for following your starred GitHub project releases",
  "type": "story",
  "url": "https://opml.bb8.fun/"
}

Anchor link Extract parameters from session

Using a similar method to extracting parameters from the configuration, Web also supports obtaining values from the session if sessions are enabled. Simply set the source property to "session" and the param property to the path to the required value:

{
  "datasource": {
    "key": "userDetails",
    "source": {
      "endpoint": "1.0/users/{userId}"
    },
    "requestParams": [
      {
        "source": "session",
        "param": "user.profile._id",
        "field": "userId",
        "target": "endpoint"
      }
    ]
  }
}

Anchor link Extract parameters from chained datasource

In addition, chained datasources can now use data from the datasource they're attached to, modifying the endpoint if "target": "endpoint":

{
  "datasource": {
    "key": "chained-endpoint",
    "source": {
      "endpoint": "1.0/makes/{name}"
    },
    "chained": {
      "datasource": "global",   // the datasource we're waiting on
      "outputParam": {
        "param": "results.0.name",  // where to find the value in the "global" datasource
        "field": "name",   // the value to replace in this endpoint URL
        "target": "endpoint"
      }
    }
  }
}

Anchor link Building filters

Filter Events can be used to generate filters for datasources. They are like regular Events but are designed to return a filter before the datasource is executed. See Filter Events for more information.

Anchor link Chained datasources

It is often a requirement to query a datasource using data already loaded by another datasource. DADI Web supports this through the use of "chained" datasources. Chained datasources are executed after all non-chained datasources, ensuring the data they rely on has already been fetched.

Add a chained property to a datasource to make it reliant on data loaded by another datasource. The following datasource won't be executed until data from the books datasource is a available:

{
  "datasource": {
    "key": "books-by-author",
    "source": {
      "type": "dadiapi",
      "endpoint": "1.0/library/authors"
    },  
    "chained": {
      "datasource": "books" // the primary (non-chained) datasource that this datasource relies on
    }
  }
}

Chained datasources are not automatically added to the page. You must still include the datasource in the datasources block of your page config.

There are two ways to use query a chained datasource using previously-fetched data. One is Filter Generation and the other is Filter Replacement.

Anchor link Filter Generation

Filter Generation is used when the chained datasource currently has no filter, and it is relying on the primary datasource to provide its values.

Example: query the "authors" datasource, using the _id from the "books" datasource

"chained": {
  "datasource": "books",
  "outputParam": {
    "field": "_id", // the filter key to use for this datasource
    "param": "results.0.author_id" // the path to the value this datasource will use in it's filter
  }
}

Specifying a field and a param causes DADI Web to generate a filter for this datasource using values from the primary datasource. For example:

Results from primary datasource

{
  "results": [
    {
      "fullName": "Ernest Hemingway",
      "author_id": 1234567890
    }
  ]
}

Generated filter for chained datasource

{ "_id": 1234567890 }

Anchor link Filter Replacement

Filter Replacement allows more advanced filtering and can inject a query into an existing datasource filter.

Using the query property, Web extracts the specified value from the primary datasource (using the path from param) and injects it into the query where {param} has been left as a placeholder.

Next, Web takes the updated value of the query property and injects the whole thing into the current datasource's filter where it finds a placeholder matching the key of the chained datasource (in the example below, "{books}" is the placeholder).

"filter": ["{books}",{"$group":{"_id":{"genre":"$genre"}}}],
"chained": {
  "datasource": "books",
  "outputParam": {
    "param": "results.0.genre_id",
    "type": "Number",
    "query": {"$match":{"genre_id": "{param}"}}
  }
}

Anchor link Chained datasource configuration

Property		Description	Example
datasource		Should match the key property of the primary datasource.
outputParam
	param	The param value specifies where to locate the output value in the results returned by the primary datasource.	"results.0._id"
	field	The field value should match the MongoDB field to be queried.	"id"
	type	The type value indicates how the param value should be treated (currently only "Number" is supported).	"Number"
	query	The query property allows more advanced filtering, see below for more detail.	{}

Anchor link Chained datasource full example

On a page that displays a car make and all it's associated models, we have two datasources querying two collections, makes and models.

Collections

• makes has the fields _id and name
• models has the fields _id, makeId and name

Datasources

• The primary datasource, makes (some properties removed for brevity)

{
  "datasource": {
     "key": "makes",
     "source": {
       "endpoint": "1.0/car-data/makes"
     },
     filter: { "name": "Ford" }
   }
}

The result of this datasource will be:

{
  "results": [
    {
      "_id": "5596048644713e80a10e0290",
      "name": "Ford"
    }
  ]
}

To query the models collection based on the above data being returned, add a chained property to the models datasource specifying makes as the primary datasource:

{
  "datasource": {
     "key": "models",
     "source": {
       "endpoint": "1.0/car-data/models"
     },
      "chained": {
        "datasource": "makes",
        "outputParam": {
          "param": "results.0._id",
          "field": "makeId"
        }
      }
   }
}

In this scenario the models collection will be queried using the value of _id from the first document of the results array returned by the makes datasource.

If your query parameter must be passed to the endpoint as an integer, add a type property to the outputParam specifying "Number".

{
  "datasource": {
     "key": "models",
     "source": {
       "endpoint": "1.0/car-data/models"
     },
     "chained": {
        "datasource": "makes",
        "outputParam": {
          "param": "results.0._id",
          "type": "Number",
          "field": "makeId"
        }
      }
   }
}

Anchor link Preloading data

Web can be configured to preload data before each request. Add a block to the main configuration file like the example below, using your datasource names in place of "channels":

"data": {
  "preload": [
    "channels"
  ]
}

Anchor link Accessing preloaded data

const Preload = require('@dadi/web').Preload
const data = Preload().get('key')

Anchor link Data Providers

Loading data into the context for rendering requires a datasource. Each datasource specifies what data provider to use and any additional parameters that the data provider needs to retrieve the data.

Built-in data providers include:

• DADI API: retrieve data from an existing DADI API
• Rest API: retrieve data from miscellaneous REST APIs requiring authentication
• Remote: retrieve data from miscellaneous REST APIs
• Markdown: load data from a folder of Markdown files (or any other text file type)
• Twitter: retrive data from the Twitter API
• Wordpress: retrive data from a Wordpress API

Anchor link DADI API

Previous to 2.0 the datasource source type for connecting to a DADI API was called remote. This was changed to dadiapi to ensure clarity with the updated and repurposed Remote provider.

A typical datasource specification file would now contain the following:

"source": {
  "type": "dadiapi",
  "endpoint": "1.0/articles"
}

The default is dadiapi, so there is no requirement to specify this property when connecting to a DADI API.

Anchor link Remote

Connect to a miscellaneous API via HTTP or HTTPS. See the following file as an example:

{
  "datasource": {
    "key": "instagram",
    "name": "Grab instagram posts for a specific user",
    "source": {
      "type": "remote",
      "protocol": "http",
      "host": "instagram.com",
      "endpoint": "{user}/?__a=1"
    },
    "auth": false,
    "requestParams": [
      {
        "param": "user",
        "field": "user",
        "target": "endpoint"
      }
    ]
  }
}

Anchor link Rest API

Connect to any RestAPI, including one which requires authentication:

{
  "datasource": {
    "key": "twitter",
    "source": {
      "type": "restapi",
      "provider": "twitter",
      "endpoint": "statuses/show",
      "auth": {
        "oauth": {
          "consumer_key": "xxx",
          "consumer_secret": "xxx",
          "token": "xxx",
          "token_secret": "xxx"
        }
      }
    },
    "fields": {
      "text": 1,
      "user.screen_name": 1
    },
    "requestParams": [
      {
        "param": "tweetid",
        "field": "id",
        "target": "query"
      }
    ]
  }
}

provider can be any of the @purest/providers. For example Facebook, google, twitter.

The source section can be partly defined in the main config api block to save repetition if needed.

Anchor link Markdown

Serve content from a local folder containing text files. You can also specify the extension to grab. Web will process any Markdown formatting (with Marked) it finds automatically as well as any Jekyll-style front matter found. Any dates/times found will be processed through JavasScript’s Date() function.

{
  "datasource": {
    "source": {
      "type": "markdown",
      "path": "./workspace/posts",
      "extension": "md"
    }
  }
}

workspace/posts/somefolder/myslug.md

--
date: 2016-02-17
title: Your title here
--
Some *markdown*

When loaded becomes the following data:

{
  "attributes": {
    "date": "2016-02-17T00:00:00.000Z",
    "title": "Your title here",
    "_id": "myslug",
    "_ext": ".md",
    "_loc": "workspace/posts/somefolder/myslug.md",
    "_path": [
      "somefolder"
    ]
  },
  "original": "--\ndate: 2016-02-17\ntitle: Your title here\n--\nSome *markdown*",
  "contentText": "Some *markdown*",
  "contentHtml": "<p>Some <em>markdown</em></p>\n"
}

NB. _path will exclude the datasource source.path.

Anchor link RSS

You can grab an XML feed and access it in your templates like any other JSON source. For example:

  {
    "datasource": {
      "key": "rss",
      "name": "RSS",
      "source": {
        "type": "rss",
        "endpoint": "https://github.com/dadi/web/releases.atom"
      },
      "count": 1,
      "fields": {
        "description": 1,
        "pubDate": 1
      }
    }
  }

Anchor link Adding logic

Anchor link Events

Events are server side JavaScript functions that can add additional functionality to a page. Events can serve as a useful way to implement logic in a logic-less template.

A simple use case for Events is counting how many users have clicked on a 'Like' button. To achieve this an Event file needs to be attached to the page which contains the 'Like' button. The Event file would check the POST request body contains expected values and then perhaps increase a counter stored in a database.

The How To Guides contains an example of an Event being used to send email via SendGrid in response to user interaction.

Anchor link The Event system

The Event system in DADI Web provides developers with a way to perform tasks related to the current request, end the current request or extend the data context that is passed to the rendering engine.

• Events are executed in sequence after a page's datasources have all returned, and they have access to the data loaded by all the datasources
• Events are attached to pages in the page specification file, using the "events" array

my-web/
  workspace/
    datasources/      
    events/           
      addAuthorInformation.js      # an Event file
    pages/
      books.json            

workspace/pages/books.json

"events": [
  "addAuthorInformation"
]

An Event is declared in the following way, receiving the original HTTP request, the response, the data context and a callback function to return control back to the controller that called it:

workspace/events/example.js

const Event = function (req, res, data, callback) {
  callback(null)
}

module.exports = function (req, res, data, callback) {
  return new Event(req, res, data, callback)
}

Anchor link The data context

The data argument that an Event receives is JSON which is eventually passed to the template rendering engine once all the Datasources and Events have finished running. data may contain some or all of the following:

• metadata about the current page
• request parameters
• data loaded by all datasources that have been run before the Events started executing
• data added by previous Events
• global configuration settings

Anchor link Sample Event file

workspace/events/full-example.js

/**
 * <Event Description>
 *
 * @param {IncomingMessage} req - the original HTTP request
 * @param {ServerResponse} res - the HTTP response
 * @param {object} data - contains the data already loaded by the page's datasources and any previous events that have fired
 * @param {function} callback - call back to the controller with two arguments, `err` and the result of the event processing
 */
const Event = (req, res, data, callback) => {
  let result = {}

  if (data.hasResults('books')) {
    result = {
      title: data.books.results[0].title
    }
  }
  else {
    result = {
      title: "Not found"
    }
  }

  // return a null error and the result
  callback(null, result)
}

module.exports = function (req, res, data, callback) {
  return new Event(req, res, data, callback)
}

module.exports.Event = Event

Anchor link Global Events

In addition to attaching Events to specific pages in the application, it is possible to declare "global events" that are fired for every page. Global Events are fired at the beginning of the request cycle, before datasources and other page Events. Add a "globalEvents" section to the main configuration file:

globalEvents: [
  "eventName"
]

Anchor link Preload Events

Preload Events are similar to Global Events, in that they are fired at the beginning of the request cycle, before any data is loaded from datasources. To attach a Preload Event to a page, add a "preloadEvents" block to the page specification file:

"preloadEvents": [
  "preloadevent-one"
]

Anchor link Filter Events

Filter Events can be used to generate filters for datasources. They are like regular Events but are designed to return a filter to the datasource so it can query it's underlying source. This could be useful when needing to specify a filter for a datasource that relies on parameters that can't be determined from the request parameters (that is, req.params).

Any filter already specified by the datasource specification will be extended with the result of the filter event.

A filter event can be attached to a datasource specification using the property filterEvent. That value must match the filename of an existing event file, without it's extension:

workspace/datasources/books.json

{
  "datasource": {
    "key": "books",
    "source": {
      "type": "dadiapi",
      "endpoint": "1.0/library/books"
    },
    "count": 10,
    "sort": {},
    "filter": {"borrowed": true},
    "filterEvent": "injectCurrentDate",
    "fields": ["title", "author"]
  }
}

workspace/events/injectCurrentDate.js

const Event = function (req, res, data, callback) {
  const filter = { date: Date.now() }

  // call the callback function, passing null in the error positon 
  callback(null, filter)
}

module.exports = function (req, res, data, callback) {
  return new Event(req, res, data, callback)
}

With the above examples, the datasource instance will be modified as follows. The filter property will be extended to add a date property (from the filter event), and a new filterEventResult property is added which contains the result of executing the filter event:

filter: { borrowed: true, date: 1507566199527 },
filterEventResult: { date: 1507566199527 }

Anchor link Middleware

Middleware functions are functions that can be added to your Web application and executed in sequence for each request. Each function has access to the request object (req), the response object (res), and the next middleware function in the stack (by convention, a variable named next).

Middleware functions can:

• execute any code
• modify the request and response objects
• end the request-response cycle (by calling res.end())
• call the next middleware function in the stack (by calling next())

Note: if the currently executing middleware function does not end the request-response cycle, it must call next() to pass control to the next middleware function. Failure to do so will cause the request to hang.

Middleware functions are stored as JavaScript files in your application's middleware folder. The location of this folder is configurable but defaults to workspace/middleware.

your-project/
  config/
  workspace/
    datasources/      # datasource specifications
    events/           # event files
    middleware/       # middleware files
      log.js          # middleware file
    pages/            # page specifications

A DADI Web application can use the following types of middleware:

• Application-level middleware
• Error-handling middleware
• Built-in middleware
• Third-party middleware

Anchor link Application-level middleware

A single middleware file may contain multiple functions, or they can be split across multiple files.

You bind application-level middleware functions to the instance of the app object by using the app.use() function, optionally specifying a route that determines the requests it applies to.

A middleware function with no route will be executed on every request.

Anchor link Route-less middleware functions

The following example shows a middleware function with no route. The function is executed every time the application receives a request:

const Middleware = function (app) {
  // output the time for each request
  app.use((req, res, next) => {
    console.log('Request received at:', Date.now())
    next()
  })
}

module.exports = function (app) {
  return new Middleware(app)
}

module.exports.Middleware = Middleware

Anchor link Route-specific middleware functions

The following example shows a middleware function mounted at the /users route. The function is only executed for requests to the /users route.

const Middleware = function (app) {
  // output the time for each request
  app.use('/users', (req, res, next) => {
    console.log('Request received at:', Date.now())
    next()
  })
}

module.exports = function (app) {
  return new Middleware(app)
}

module.exports.Middleware = Middleware

Anchor link HTTP method restriction

To restrict a middleware function to only certain HTTP methods, you can test the current request's method and call next() if it doesn't match:

app.use('/users', (req, res, next) => {
  if (req.method.toLowercase() !== 'get') {
    return next()
  }

  console.log('GET request received at:', Date.now())
  return next()
})

Anchor link Error-handling middleware

Error-handling middleware functions must accept four arguments. Without the additional argument (err) the function will be interpreted as regular middleware and won't handle errors.

Define error-handling middleware functions in the same way as other middleware functions, except with four arguments instead of three:

app.use((err, req, res, next) => {
  console.error(err.stack || err)
  res.end(500, 'Server error!')
})

Anchor link Built-in middleware

DADI Web has some built-in middleware functions, which in some cases can be turned on or off using the main configuration file.

Type	Description
Body parser	parses the body of an incoming request and makes the data available as the property req.body
Caching	determines if the current request can be handled by a previously cached response
Compression	compresses the response before sending
Request logging	logs every request to a file
Sessions	handles session data
Static files	serves static assets from the public folder, such as JavaScript, CSS, HTML files, images, etc
Virtual directories	serves content from configured directories not handled by the existing page/route specifications

Note: the body parser middleware can handle JSON, raw, plain text and URL-encoded request bodies. It does not handle multipart bodies due to their complex and typically large nature. For multipart bodies, try one of the following modules: busboy, multiparty, formidable or multer

Anchor link Third-party middleware

You can add third-party middleware to your DADI Web application to add new functionality that DADI Web doesn't have built-in. Simply install the Node.js module for the required functionality and load it in an application-level middleware function.

The following example shows how to use the module online to track online user activity using Redis:

$ npm install online
$ npm install redis

const Online = require('online')
const redis = require('redis')
const redisClient = redis.createClient()

// use the Redis client
const online = Online(redisClient)

const Middleware = function (app) {
  // executes for every request
  app.use((req, res, next) => {
    // add a call to track current user's activity, assuming a `user` object in the request
    online.add(req.user.id, (err) => {
      if (err) {
        return next(err) // call an error-handling middleware function
      }

      next() // calls the next middleware function (defined below)
    })
  })

  // executes for every request
  app.use((req, res, next) => {
    // get users active in the last 10 minutes
    online.last(10, (err, ids) => {
      if (err) {
        return next(err) // call an error-handling middleware function
      }

      console.log('Users online:', ids)

      // call next middleware function
      next()
    })
  })
}

module.exports = function (app) {
  return new Middleware(app)
}

module.exports.Middleware = Middleware

Anchor link Middleware template

/**
 * workspace/middleware/example.js
 */
const Middleware = function (app) {
  // execute for every request
  app.use((req, res, next) => {
    console.log(req.url)
    console.log(req.params)
    next()
  })

  // route-mounted, execute for requests to /channel
  app.use('/channel', (req, res, next) => {
    console.log('Channel params:', JSON.stringify(req.params))
    next()
  })

  // error handler
  app.use((err, req, res, next) => {
    console.log('Error:', err)
    next()
  })
}

module.exports = function (app) {
  return new Middleware(app)
}

module.exports.Middleware = Middleware

Anchor link Post processors

Web "Post Processors" give the ability to manipulate the raw output of the template engine, before the page is cached and served. Similar to the Wordpress feature, it is useful for linting (thinking RSS feeds in particular), minifying and formatting.

Post processors are located in the workspace folder. The default location is workspace/processors, however this is configurable using the paths.processors configuration setting.

Post processors can be defined globally for the application or on a page-by-page basis. They are loaded in order of definition, globally defined ones first, and the output from each is passed to the next so you can chain functions.

Anchor link Example usage: minifying HTML

Anchor link Add globalPostProcessors to configuration file

"globalPostProcessors": [
    "minify-html"
]

Anchor link Add processor file

This processor uses the package html-minifier

/workspace/processors/minify-html.js

const minify = require('html-minifier').minify

module.exports = (data, output) => {
  return minify(output, {
    collapseWhitespace: true,
    minifyCSS: true,
    minifyJS: true,
    removeRedundantAttributes: true,
    useShortDoctype: true,
    removeAttributeQuotes: true
  })
}

Note:

• the page json data object is also passed to the function
• in addition it can be defined or disabled at page level in the page.settings block of any page specification file -- advice

Anchor link Page-specific processors

To add a post processor to a page add a postProcessors property to the page's settings block. Page-specific post processors are run after globally defined post processors.

"settings": {
  "postProcessors": ["remove-swear-words"]
}

Anchor link Disabling post processors for a page

To disable post processors for a page add a postProcessors property to the page's settings block, and set this to false.

"settings": {
  "postProcessors": false
}

Anchor link Upgrading from Web 4.0

Previous versions of Web had a built-in post processor: beautify-html. It was configured using the page configuration option page.settings.beautify. To enable the same functionality in Web 5.0 and greater, follow the guide below.

Anchor link Install beautify package

npm install js-beautify

Anchor link Add new post processor file workspace/processors/beautify-html.js

const beautifyHtml = require('js-beautify').html

module.exports = (data, output) => {
  return beautifyHtml(output)
}

Anchor link Modify page configuration

existing page.json file example

{
  "page": {
    "name": "Sitemap page",
    "description": "Sitemap",
    "language": "en"
  },
  "settings": {
    "cache": true,
    "beautify": true
  },
  ...
}

new page.json file example

{
  "page": {
    "name": "Sitemap page",
    "description": "Sitemap",
    "language": "en"
  },
  "settings": {
    "cache": true,
    "postProcessors": [
      "beautify-html"
     ]
  },
  ...
}

Anchor link Internationalization

Adding support for internationalization in your application is relatively easy when using DADI API datasources to fetch data. API 4.2 (and above) has built-in support for documents containing fields with multiple language variations.

You must be using API 4.2 or above for this feature. See the documentation for API 4.2 for more detailed information on internationalization.

Anchor link Requesting a specific language

To retrieve your content in a specific language in Web, you simply need to modify your datasource to include a lang parameter in the query string. For example, the datasource endpoint /1.0/library/books?lang=fr will return data from API with the French versions of fields, if they exist:

{
  "_id": "58176e72bafa53b625aebd4f",
  "_i18n": {
    "title": "fr",
    "author": "en"
  },
  "title": "Le Petit Prince",
  "author": "Antoine de Saint-Exupéry"
}

Note that any fields that don't have a translation matching the specified lang parameter will be returned in the default language.

Anchor link Dynamically selecting a language

To dynamically select a language parameter to send to API, it is possible to pass a parameter from the URL. There are two options for this: using a dynamic URL parameter or using the query string.

In both cases, when using the DADI API provider in a datasource, the value of the lang parameter will be automatically added to the datasource's endpoint when making the request.

The lang parameter is added automatically; if you don't want to pass the lang parameter you can explicitly disable it using the i18n property in the datasource specification:

{
  "datasource": {
    "key": "articles",
    "source": {
      "type": "dadiapi",
      "endpoint": "1.0/library/articles"
    },
    "count": 4,
    "paginate": true,
    "i18n": false
  }
}

Anchor link Dynamic URL parameter

In this case, you would have set up a page route to include a lang portion, for example:

{
  "page": {
    "key": "article"
  },
  "datasources": [
    "articles"
  ],
  "routes": [
    {
      "path": "/:lang/:title"
    }
  ]
}

This puts the value of the lang parameter into the request parameters, making it available in req.params when using Events. A URL such as /fr/about would populate req.params like this:

{
  "lang": "fr"
}

Anchor link Query string

Using the query string, you can put the lang parameter into the URL like so: /about?lang=fr would again populates req.params like this:

{
  "lang": "fr"
}

Anchor link Unfiltered response

If you don't pass a language parameter to API, the response will contain the raw content of documents, containing the original value and all the language variations of each translatable field. In this case, no _i18n field is added to the documents.

{
  "_id": "58176e72bafa53b625aebd4f",
  "title": "The Little Prince",
  "title:pt": "O Principezinho",
  "title:fr": "Le Petit Prince",
  "author": "Antoine de Saint-Exupéry"
}

Anchor link Performance

Anchor link Caching

Caching is enabled by default, but disabled when ”debug”: true. You can configure the cache headers for each MIME type, or disable the cache entirely in your config:

"caching" {
    "directory": {
        "enabled: false
    }
}

Anchor link Compression

Web supports gzip and br (Brotli) compression and is on for all supported file-types by default. You can disable it globally:

"headers" {
    "useCompression": {
        "enabled: false
    }
}

Anchor link Cache flush

DADI Web has a cache invalidation endpoint which allows an authorised user to flush the cache for either a specific path or the entire application. This process clears both page (HTML) and datasource (JSON) cache files.

The user must send a POST request to /api/flush with a request body containing 1) the path to flush, and 2) a set of credentials that match those held in the configuration file's auth block:

Anchor link Flush all cache files

POST /api/flush HTTP/1.1
Host: www.example.com

{ "path": "*", "clientId": "your-client-id", "secret": "your-secret" }

Anchor link Flush cache files for a specific path

POST /api/flush HTTP/1.1
Host: www.example.com

{ "path": "/books/crime", "clientId": "testClient", "secret": "superSecret" }

Anchor link Serving static assets and content

Anchor link Public folder

The public folder is where you can store any static files you may need to serve to a browser (e.g., CSS, JavaScript, video files etc), the path can be configured to any location you like.

Content in this folder obeys your useCompression and cacheControl settings.

Anchor link Virtual directories

Virtual directories a similar to the public folder but are particularly geared to serving static content. You can list as many as you require in your config as an array:

"virtualDirectories": [
    {
        "path": "data/legacy_features",
        "index": "default.html",
        "forceTrailingSlash": false
    }
]

index is a similar function to a traditional web server where hitting the root of a folder will serve that document without a URI e.g., /legacy_features/ will serve /legacy_features/default.html to the browser.

index and forceTrailingSlash are both optional.

Anchor link Debugging

Anchor link Log formatting

You can format the logs for easier readability by using Bunyan:

npm install -g bunyan

Then start the app:

npm start | bunyan -o short

Anchor link Debug view

When the config option is set to true you can append ?debug to any DADI Web URL and you will see how Web constructed that page.

This will look similar to the following:

From here you can see how to construct you templates to output specific variable or loop over particular objects. It is also useful for seeing the output of any Events you have which may output values into the page.

Anchor link Security

Anchor link CSRF tokens

Cross-Site Request Forgery (CSRF) is an attack that forces an end user to execute unwanted actions on a web application in which they're currently authenticated. CSRF attacks specifically target state-changing requests, not theft of data, since the attacker has no way to see the response to the forged request. More about CSRF.

CSRF protection allows developers to use a per-request CSRF token which will be injected into the view model, and ensures that all POST requests supply a correct CSRF token. Without a correct token, with CSRF enabled, users will be greeted with a 403.

To enable CSRF, set the security.csrf config option in your config/config.{env}.json file:

"security": {
  "csrf": true
}

Once enabled, the variable csrfToken will be injected into the viewModel. You will need to add this to any forms which perform a POST using the field name _csrf, like so:

<form action="/" method="post">
  <input type="text" name="test_input_safe">
  <input type="hidden" name="_csrf" value="{csrfToken}">
  <input type="submit" value="Submit form">
</form>

If the CSRF token provided is incorrect, or one isn't provided, then a 403 forbidden error will occur.

A working example can be found here: dadi-web-csrf-test.

Anchor link SSL

To use SSL you first need to generate the necessary certificates. This will vary depending on your platform. Digital Ocean has a useful introduction that may assist.

After that you need to tell Web where to find your SSL files and enable http in your config.json file.

"server": {
  "host": "127.0.0.1",
  "port": 443,
  "protocol": "https",
  "sslPassphrase": "superSecretPassphrase",
  "sslPrivateKeyPath": "keys/server.key",    
  "sslCertificatePath": "keys/server.crt"
}

Anchor link Application status

DADI Web has an endpoint which returns a JSON object containing information about an application's platform, process and health state. The information returned by the endpoint includes:

• Latest version of the DADI Web application
• Node.js version
• Process ID
• Process uptime
• Process memory usage
• System host name
• System platform and version
• System uptime
• Memory (free and total)
• Current load averages

Anchor link Health Check Routes

In addition to system information, if any routes are specified in the configuration Web will send a request to each one and return data about the response. Each route is a JSON object containing the URL to hit and the expected response time in seconds:

{
  "route": "/movies/latest",
  "expectedResponseTime": 1
}

The response will contain a block for each of the routes configured, along with the HTTP status that was received, the response time, and a colour value indicating how the response performed against the configured response time.

"routes": [
  {
    "route": "/movies/latest",
    "status": 200,
    "responseTime": 0.039,
    "healthStatus": "Green"
  }
]

Anchor link Configuration

To enable the status endpoint, add a configuration block:

{
  "status": {
    "routes": [
      {
        "route": "/movies/latest",
        "expectedResponseTime": 1
      }
    ]
  }
}

Anchor link User-Agent identifier

The following User-Agent header is used when making health check requests: 'User-Agent': '@dadi/status'

Anchor link Status request

Send a POST request to /api/status with a request body containing a set of credentials that match those held in the configuration file's auth block:

POST /api/status HTTP/1.1
Host: www.example.com
Content-Type: application/json

{ "clientId": "your-client-id", "secret": "your-secret" }

Anchor link Status response

With the configuration given above, expect a response similar to the following:

{
  "service": {
    "site": "Your Web Application",
    "package": "@dadi/web",
    "versions": {
      "current": "6.0.0",
      "latest": "6.0.1"
    }
  },
  "process": {
    "pid": 19463,
    "uptime": 3.523,
    "uptimeFormatted": "0 days 0 hours 0 minutes 3 seconds",
    "versions": {
      "http_parser": "2.3",
      "node": "0.12.0",
      "v8": "3.28.73",
      "uv": "1.0.2",
      "zlib": "1.2.8",
      "modules": "14",
      "openssl": "1.0.1l"
    }
  },
  "memory": {
    "rss": "86.508 MB",
    "heapTotal": "65.771 MB",
    "heapUsed": "32.938 MB"
  },
  "system": {
    "platform": "darwin",
    "release": "14.5.0",
    "hostname": "hudson",
    "memory": {
      "free": "37.781 MB",
      "total": "8.000 GB"
    },
    "load": [
      2.2958984375,
      2.27197265625,
      2.25927734375
    ],
    "uptime": 155084,
    "uptimeFormatted": "1 days 19 hours 4 minutes 44 seconds"
  },
  "routes": [
    {
      "route": "/movies/latest",
      "responseTime": 2,
      "healthStatus": "Amber"
    }
  ]
}

Anchor link How-to guides

Anchor link Migrating from version 2.x to 3.x

1. Install Dust.js dependency

Web 3.0 supports multiple template engines. As a consequence, Dust.js is now decoupled from core and needs to be included as a dependency on projects that want to use it.

npm install @dadi/web-dustjs --save

2. Change bootstrap script

The bootstrap script (which you may be calling index.js, main.js or server.js) now needs to inform Web of the engines it has available and which npm modules implement them.

require('@dadi/web')({
  engines: [
    require('@dadi/web-dustjs')
  ]
})

3. Update config

The dust config block has been moved inside a generic engines block.

Before:

"dust": {
  "cache": true,
  "debug": true,
  "debugLevel": "DEBUG",
  "whitespace": true,
  "paths": {
    "helpers": "workspace/utils/helpers"
  }
}

After:

"engines": {
  "dust": {
    "cache": true,
    "debug": true,
    "debugLevel": "DEBUG",
    "whitespace": true,
    "paths": {
      "helpers": "workspace/utils/helpers"
    }
  }
}

4. Move partials directory

Before Web 3.0, Dust templates were separated between the pages and partials directories, with the former being used for templates that generate a page (i.e. have a route) and the latter being used for partials/includes.

In Web 3.0, all templates live under the same directory (pages). The distinction between a page and a partial is made purely by whether or not the template has an accompanying JSON schema file.

Also, pages and partials can now be located in sub-directories, nested as deeply as possible.

To migrate an existing project, all you need to do is move the partials directory inside pages and everything will work as expected.

Before:

workspace
|_ pages
|_ partials

After:

workspace
|_ pages
  |_ partials

mv workspace/partials workspace/pages

5. Update Dust helpers

If your project is using custom helpers, you might need to change the way they access the Dust engine. You should now access the module directly, rather than reference the one from Web.

// Before
var dust = require('@dadi/web').Dust
require('@dadi/dustjs-helpers')(dust.getEngine())

// After
var dust = require('dustjs-linkedin')
require('@dadi/dustjs-helpers')(dust)

Anchor link Dealing with form data and SendGrid

This is an example of an Event which uses SendGrid to send a message from an HTML form.

workspace/pages/contact.dust

{?mailResult}<p>{mailResult}</p>{/mailResult}

<form action="/contact/" method="post">
  <p>
    <label class="hdr" for="name">Name</label>
    <input autofocus id="name" name="name" placeholder="Your full name" class="normal" type="text">
  </p>
  <p>
    <label class="hdr" for="email">Email</label>
    <input id="email" name="email" required placeholder="Your email address" class="normal" type="email">
  </p>
  <p>
    <label class="hdr" for="phone">Phone</label>
    <input id="phone" name="phone" placeholder="Contact telephone number" class="normal" type="text">
  </p>
  <p>
    <label class="hdr" for="message">Message</label>
    <textarea style="min-height:166px" rows="5" id="message" name="message" required placeholder="What do you want to talk about?" class="normal" type="email"></textarea>
  </p>
  <p>
    <button type="submit">Send message</button>
  </p>
</form>

You need an API key from SendGrid to use this Event in your application. Once you have obtained an API key from SendGrid.com, DADI Web should be started with an environment variable containing the API key. You will also need to whitelist your IP address within the SendGrid dashboard.

You could hardcode your API key, but be careful not to commit the code to a publicly accessible GitHub repo.

Starting DADI Web with an environment variable

$ SENDGRID_API=71713987-9f01-4dea-b3d4-8d0bcd9d53ed node index.js

workspace/events/contact.js

const sendgrid = require('sendgrid')(process.env['SENDGRID_API'])

const Event = function (req, res, data, callback) {
  // On form post
  switch (req.method.toLowerCase()) {
    case 'post':
      // Validate the inputs
      if (req.body.email && isEmail(req.body.email) && req.body.message) {
        const message = "Name: " +req.body.name + "\n\nEmail: " + req.body.email + "\n\nPhone: " + req.body.phone + "\n\nMessage:\n\n" + req.body.message

        var request = sendgrid.emptyRequest({
          method: 'POST',
          path: '/v3/mail/send',
          body: {
            personalizations: [{
              to: [{
                email: 'hello@dadi.cloud',
              }],
              subject: '[dadi.cloud] Contact form message',
            }],
            from: {
              email: 'hello@dadi.cloud',
            },
            content: [{
              type: 'text/plain',
              value: message,
            }]
          }
        })

        sendgrid.API(request, (error, response) => {
          if (error) {
            data.mailResult = 'There was a problem sending the email.'
          } else {
            data.mailResult = 'Thank you for your message, you will hear back from us soon.'
          }

          callback()
        })
      } else {
        data.mailResult = 'All fields are required.'
        callback()
      }

    break
  default:
      return callback()
  }

}

// Taken from: http://stackoverflow.com/a/46181/306059
function isEmail(email) {
  const re = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/

  return re.test(email)
}

module.exports = function (req, res, data, callback) {
  return new Event(req, res, data, callback)
}