Handling AJAX Calls With Node.js and Express (Part 4)

Articles in the series:

If you’ve been following along with this series, you should have a basic application for searching and scraping Craigslist for jobs in San Francisco. The end goal is to have an application that users can login to, then search for jobs. From there the end user can either apply for jobs or save jobs they may be interested in.

Before adding any additional functionality, we need to refactor the code a bit by moving some code out of app.js and into separate modules so that the entire app is more modular.

Configuration

First, move the config settings into a separate file, outside the main project. It’s always a good idea to separate configuration from actual code so that other users who wish to use your project can easily make it their own by quickly adding their own configuration.

Create a config.js file and add the following code:

1
2
3
4
5
6
7
module.exports = {
  google: {
    returnURL: 'http://127.0.0.1:3000/auth/google/callback',
    realm: 'http://127.0.0.1:3000'
  },
  mongoUrl: 'mongodb://localhost/craigslist'
};

Then make sure to include the file as part of app.js’s dependencies:

1
var config = require('./config');

Finally, update these two areas within app.js:

1
2
// connect to the database
mongoose.connect(config.mongoUrl);

And:

1
2
3
4
passport.use(new GoogleStrategy({
  returnURL: config.google.returnURL,
  realm: config.google.realm
},

User Model

Next, update the user schema for mongoose.

Create a new folder called “models” and add a file called user.js to hold the user schema:

1
2
3
4
5
6
7
8
9
10
11
12
var mongoose = require('mongoose');
var config = require('../config');

console.log(config);

// create a user model
var userSchema = new mongoose.Schema({
  name: String,
  email: {type: String, lowercase: true }
});

module.exports = mongoose.model('User', userSchema);

Add this to the dependencies:

1
var user = require('./models/user');

Then update app.js:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
// passport settings
passport.serializeUser(function(user, done) {
  console.log('serializeUser: ' + user.id)
  done(null, user.id);
});
passport.deserializeUser(function(id, done) {
  user.findOne({_id : id}, function(err, user) {
    console.log(user)
    if(!err) done(null, user);
    else done(err, null)
  });
});

passport.use(new GoogleStrategy({
  returnURL: config.google.returnURL,
  realm: config.google.realm
},
  function(identifier, profile, done) {
    console.log(profile.emails[0].value)
    process.nextTick(function() {
      var query = user.findOne({'email': profile.emails[0].value});
      query.exec(function(err, oldUser) {
        if(oldUser) {
          console.log("Found registered user: " + oldUser.name + " is logged in!");
          done(null, oldUser);
        } else {
          var newUser = new user();
          newUser.name = profile.displayName;
          newUser.email = profile.emails[0].value;
          console.log(newUser);
          newUser.save(function(err){
            if(err){
              throw err;
            }
            console.log("New user, " + newUser.name + ", was created");
            done(null, newUser);
          });
        }
      });
    });
  }
));

The Passport code searches the database to see if a user already exists before creating a new one - which is no different from last time. However, see if you can dig a bit deeper and see the subtle differences.

Routes

Next, move the main routing into a separate module by adding the following code to routes/index.js:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
var request = require('request');

exports.index = function(req, res){
  res.render('index', { user: req.user });
};

exports.search = function(req, res) {
  res.render('search', { user: req.user.name });
};

exports.searching = function(req, res){
  // input value from search
  var val = req.query.search;
  // url used to search yql
  var url = "http://query.yahooapis.com/v1/public/yql?q=select%20*%20from%20craigslist.search" +
  "%20where%20location%3D%22sfbay%22%20and%20type%3D%22jjj%22%20and%20query%3D%22" + val + "%22&format=" +
  "json&diagnostics=true&env=store%3A%2F%2Fdatatables.org%2Falltableswithkeys";

  requests(url,function(data){
    res.send(data);
  });
};

function requests(url, callback) {
  // request module is used to process the yql url and return the results in JSON format
  request(url, function(err, resp, body) {
    var resultsArray = [];
    body = JSON.parse(body);
    // console.log(body.query.results.RDF.item)
    // logic used to compare search results with the input from user
    if (!body.query.results.RDF.item) {
      results = "No results found. Try again.";
      callback(results);
    } else {
      results = body.query.results.RDF.item;
      for (var i = 0; i < results.length; i++) {
        resultsArray.push(
          {title:results[i].title[0], about:results[i]["about"], desc:results[i]["description"]}
        );
      };
    };
    // pass back the results to client side
    callback(resultsArray);
  });
};

Again, add the dependency: var routes = require('./routes');

The routes section in app.js should now look like this:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
// user routes
app.get('/', routes.index);
app.get('/search', ensureAuthenticated, routes.search);
app.get('/searching', ensureAuthenticated, routes.searching);
app.get('/logout', function(req, res){
  req.logOut();
  res.redirect('/');
});

// auth routes
app.get('/auth/google',
  passport.authenticate('google'),
  function(req, res){
});
app.get('/auth/google/callback',
passport.authenticate('google', { failureRedirect: '/' }),
  function(req, res) {
    res.redirect('/search');
  }
);

// test authentication
function ensureAuthenticated(req, res, next) {
  if (req.isAuthenticated()) { return next(); }
  res.redirect('/')
}

Passport

Now, move the main authentication code to a separate file.

Create a new file called authentication.js and add the following code:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
// authentication 

var passport = require('passport')
var GoogleStrategy = require('passport-google').Strategy;
var config = require('./config');
var user = require('./models/user');

// passport settings
passport.serializeUser(function(user, done) {
  console.log('serializeUser: ' + user.id)
  done(null, user.id);
});
passport.deserializeUser(function(id, done) {
  user.findOne({_id : id}, function(err, user) {
    console.log(user)
    if(!err) done(null, user);
    else done(err, null)
  });
});

passport.use(new GoogleStrategy({
  returnURL: config.google.returnURL,
  realm: config.google.realm
},
  function(identifier, profile, done) {
    console.log(profile.emails[0].value)
    process.nextTick(function() {
      var query = user.findOne({'email': profile.emails[0].value});
      query.exec(function(err, oldUser) {
        if(oldUser) {
          console.log("Found registered user: " + oldUser.name + " is logged in!");
          done(null, oldUser);
        } else {
          var newUser = new user();
          newUser.name = profile.displayName;
          newUser.email = profile.emails[0].value;
          console.log(newUser);
          newUser.save(function(err){
            if(err){
              throw err;
            }
            console.log("New user, " + newUser.name + ", was created");
            done(null, newUser);
          });
        }
      });
    });
  }
));

module.exports = passport;

Then back in app.js, make sure to import that module back in by adding it as a dependency:

1
var passport = require('./authentication');

Fire up the server, and test your app out. If it all went well, everything should still work properly.

Finally, let’s update the styles.

Styles

First, add in a Bootstrap stylesheet to the layout.jade file:

1
link(rel='stylesheet', href='//netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css')

index.jade

1
2
3
4
5
6
7
8
9
10
11
12
extends layout

block content
    h1 Search Login
    .lead Please login to search
    br
    form(METHOD="LINK", ACTION="/auth/google")
        input(type="submit", value="Login with Google", class='btn btn-large btn-primary')

    script(src="//cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js")
    script(src="//cdnjs.cloudflare.com/ajax/libs/handlebars.js/1.0.0/handlebars.min.js")
    script(src="/javascripts/main.js")

search.jade

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
extends layout

block content
    h1 Search SF Jobs
    .lead Welcome, #{user}
    form(METHOD="LINK", ACTION="logout")
        input(type="submit", value="Logout", class='btn btn-sm btn-primary')
    br
    br
    input#search(type="search", placeholder="search...")
    br
    br
    ul#results
    include template.html

    script(src="//cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js")
    script(src="//cdnjs.cloudflare.com/ajax/libs/handlebars.js/1.0.0/handlebars.min.js")
    script(src="/javascripts/main.js")

Wait? How did we capture the user’s name? Go back and look at the /searching route.

Looks a little better. :)

part-4

Alright, next time we’ll expand the app’s functionality to allow users to save jobs they may be interested in applying to at a later date. Until then, check out the latest code here. Cheers!

Node Twitter Sentiment - Part 2

This is for the Node-js-Denver-Boulder Meetup <3 Cheers!

Miss part 1? Check it out here.

Let’s begin …

Before adding additional functionality to the Node Twitter Sentiment Analysis application, we need to refactor the code. Frankly, there are some mistakes that were made on purpose to highlight an issue that many new developers overlook when first working with Node.

Remember this function from index.js in the routes folder:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
exports.search = function(req, res) {
  // grab the request from the client
  var choices = JSON.parse(req.body.choices);
  // grab the current date
  var today = new Date();
  // establish the twitter config (grab your keys at dev.twitter.com)
  var twitter = new twit({
    consumer_key: config.consumer_key,
    consumer_secret: config.consumer_secret,
    access_token: config.access_token,
    access_token_secret: config.access_token_secret
  });
  // set highest score
  var highestScore = -Infinity;
  // set highest choice
  var highestChoice = null;
  // create new array
  var array = [];
  // set score
  var score = 0;
  console.log("----------")

  // iterate through the choices array from the request
  for(var i = 0; i < choices.length; i++) {
    (function(i) {
    // add choice to new array
    array.push(choices[i])
    // grad 20 tweets from today
    twitter.get('search/tweets', {q: '' + choices[i] + ' since:' + today.getFullYear() + '-' +
      (today.getMonth() + 1) + '-' + today.getDate(), count:20}, function(err, data) {
        // perform sentiment analysis (see below)
        score = performAnalysis(data['statuses']);
        console.log("score:", score)
        console.log("choice:", choices[i])
        //  determine winner
        if(score > highestScore) {
          highestScore = score;
          highestChoice = choices[i];
          console.log("winner:",choices[i])
        }
        console.log("")
      });
    })(i)
  }
  // send response back to the server side; why the need for the timeout?
  setTimeout(function() { res.end(JSON.stringify({'score': highestScore, 'choice': highestChoice})) }, 5000);
};

Essentially we’re grabbing the user inputted data, pulling tweets based on the inputs, and then calculating the sentiment of those tweets. The timeout is necessary because of how Node works. Because Node is asynchronous, functions do not block other functions from running. Without the 5 second time-out, the next function will append the results to the DOM without waiting for the function to finish running. Essentially, nothing is appended. Make sense?

Put another way, when functions run that are blocking, they wait there for the result to come back before another function fires. Node, on the other hand, will continue executing the code that comes after it (because it’s functions are non-blocking(, then jump back when the result is available.

So, why won’t a timeout work then?

Again, the code has a function that sends the results in 5 seconds, regardless as to the execution state of the call to twitter. What happens though, if we run the program without a network connection? Or if Twitter is down? Or if we pulled in 10,000 tweets instead of 20?

It’s still going to return results after 5 seconds. This is not what we want, obviously. So, how do we fix it? There’s a number of different methods, none of which fully solve it in an elegant manner. In this post, we’ll look at:

Method URL Library
Async node-twitter-sentiment-async https://github.com/caolan/async
Promises node-twitter-sentiment-promises https://github.com/kriskowal/q
Generators n/a n/a
IcedCoffeeScript n/a https://github.com/maxtaco/coffee-script

Async

Thanks to Manish Vachharajani for developing the code for this example.

One solution is to use the Async. This is often the go-to solution, since the syntax is simple, it’s totally straightforward, and it uses call backs. In fact, in order to use Async, you must follow the convention of providing the callback as the last argument of the Async function. Thus, for users used to callbacks, this is an extremely easy solution.

Basics

Start by installing the package:

1
$ npm install async

In our code we will be using the map() helper method, which takes an array, a filter function, and a callback. The filter function is an async function that takes a callback.

Simple example:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
var async = require('async');

var names = ["michael","richard","john","jennifer","ben","julie"];

async.map(names, getInfo, function (err, result) {
if(!err) {
  console.log('Finished: ' + result);
} else {
  console.log('Error: ' + err);
}

});

function getInfo(name, callback) {
setTimeout(function() {
  callback(null, name.toUpperCase());
}, 1000);
}

Test it out here.

Basically, we have an array of names, in lower case, which we are converting to uppercase, then outputting via a console.log. Let’s say that another function depended on the results of getInfo, if getInfo was long-running, then the other function could fire before getInfo returned the results. Thus, the need to suspend the function until the results are returned.

Update Node-Twitter-Sentiment

We just need to update the index.js file in the “routes” folder:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"use strict";

var path = require("path");
var twit = require('twit');
var sentimental = require('Sentimental');
var config = require("../config");
var async = require('async');

exports.index = function(req, res){
  res.render('index', { title: "Twit-Decision"});
};

exports.ping = function(req, res){
  res.send("pong!", 200);
};

exports.search = function(req, res) {
  // grab the request from the client
  var choices = JSON.parse(req.body.choices);
  // grab the current date
  var today = new Date();
  // establish the twitter config (grab your keys at dev.twitter.com)
  var twitter = new twit({
    consumer_key: config.consumer_key,
    consumer_secret: config.consumer_secret,
    access_token: config.access_token,
    access_token_secret: config.access_token_secret
  });
  console.log("----------")

  // grade 20 tweets from today with keyword choice and call callback
  // when done
  function getAndScoreTweets(choice, callback) {
    twitter.get('search/tweets', {q: '' + choice + ' since:' + today.getFullYear() + '-' +
      (today.getMonth() + 1) + '-' + today.getDate(), count:20}, function(err, data) {
        // perform sentiment analysis (see below)
      if(err) {
        console.log(err);
        callback(err.message, undefined);
        return;
      }
      var score = performAnalysis(data['statuses']);
      console.log("score:", score)
      console.log("choice:", choice)
      callback(null, score);
    });
  }
  //Grade tweets for each choice in parallel and compute winner when
  //all scores are collected
  async.map(choices, getAndScoreTweets, function(err, scores) {
    if(err) {
      console.log("Unable to score all tweets");
      res.end(JSON.stringify(err));
    }
    var highestChoice = choices[0];
    var highestScore = scores.reduce(function(prev, cur, index) {
      if(prev < cur) {
        highestChoice = choices[index];
        return cur;
      } else {
        return prev;
      }
    });
    res.end(JSON.stringify({'score': highestScore, 'choice': highestChoice}));
  });
}

function performAnalysis(tweetSet) {
  //set a results variable
  var results = 0;
  // iterate through the tweets, pulling the text, retweet count, and favorite count
  for(var i = 0; i < tweetSet.length; i++) {
    var tweet = tweetSet[i]['text'];
    var retweets = tweetSet[i]['retweet_count'];
    var favorites = tweetSet[i]['favorite_count'];
    // remove the hashtag from the tweet text
    tweet = tweet.replace('#', '');
    // perform sentiment on the text
    var score = sentimental.analyze(tweet)['score'];
    // calculate score
    results += score;
    if(score > 0){
      if(retweets > 0) {
        results += (Math.log(retweets)/Math.log(2));
      }
      if(favorites > 0) {
        results += (Math.log(favorites)/Math.log(2));
      }
    }
    else if(score < 0){
      if(retweets > 0) {
        results -= (Math.log(retweets)/Math.log(2));
      }
      if(favorites > 0) {
        results -= (Math.log(favorites)/Math.log(2));
      }
    }
    else {
      results += 0;
    }
  }
  // return score
  results = results / tweetSet.length;
  return results
}

What’s going on?

Let’s look at the specific changes:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
// grade 20 tweets from today with keyword choice and call callback
// when done
function getAndScoreTweets(choice, callback) {
  twitter.get('search/tweets', {q: '' + choice + ' since:' + today.getFullYear() + '-' +
    (today.getMonth() + 1) + '-' + today.getDate(), count:20}, function(err, data) {
      // perform sentiment analysis (see below)
    if(err) {
      console.log(err);
      callback(err.message, undefined);
      return;
    }
    var score = performAnalysis(data['statuses']);
    console.log("score:", score)
    console.log("choice:", choice)
    callback(null, score);
  });
}
//Grade tweets for each choice in parallel and compute winner when
//all scores are collected
async.map(choices, getAndScoreTweets, function(err, scores) {
  if(err) {
    console.log("Unable to score all tweets");
    res.end(JSON.stringify(err));
  }
  var highestChoice = choices[0];
  var highestScore = scores.reduce(function(prev, cur, index) {
    if(prev < cur) {
      highestChoice = choices[index];
      return cur;
    } else {
      return prev;
    }
  });
  res.end(JSON.stringify({'score': highestScore, 'choice': highestChoice}));
});

We pass in the choices array, the getAndScoreTweets() function (which handles the calculating of sentiment), then the results are serialized and sent back to the client. async.map() suspends the getAndScoreTweets() function until it’s done running. Thus, the results are not sent back to the client until Sentiment is done.

Further, async.map() allows you to do a long delay operation on each array element because of the fact that the mapped function must call “callback” - which happens in the getInfo() function.

Simple, right?

Check out the final code here: https://github.com/mjhea0/node-twitter-sentiment-async

Promises

Thanks to Richard Lucas for developing the code and writing the following explanation.

Promises are not the easiest JavaScript concept to wrap your head around, so do not feel bad if this concept takes time to understand. It certainly has taken a lot of time for myself, and I still get caught up and confused in using some of the methods. In this example, I tried to just use a simple (and hopefully easy to understand) pattern of deferreds using the Q promise library. You may also have experience with jQuery deferreds via the $.Deferred object. They are very similar.

What are promises (from the Q documentation)

If a function cannot return a value or throw an exception without blocking, it can return a promise instead. A promise is an object that represents the return value or the thrown exception that the function may eventually provide. A promise can also be used as a proxy for a remote object to overcome latency.

Here are some great resources for learning more about promises

  1. Promises A+ Spec
  2. Q Library
  3. Promisesjs.org - Great introduction
  4. Promises by Nodeschool.io
  5. Javascript Promises in Wicked Detail
  6. Promises in Node.js
  7. Using Promises with Q
  8. Using jQuery Deferreds - Book from O’Reilly

Pattern used

Here the deferred pattern was used, which goes something like this:

1
2
3
4
5
6
7
8
9
var promise = function(err, result) {
  var deferred = Q.defer();
  if (err) {
    deferred.reject(new Error(err));
  } else {
    deferred.resolve(result);
  }
  return deferred.promise;
}

You then go on to using the then and done methods:

1
2
3
4
5
promise.then(function(data) {
  return doSomething(data);
}).done(function(data) {
  return finishSomething(data);
});

How they were implemented

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
var searchTweets = function(choice) {
  var deferred = Q.defer(), // declare the deferred
     ...

  twitter.get('search/tweets',
  {
    q: '' + choice + ' since:' + dateString,
    count: 20
  }, function(err, data) {
      if (err) {
        deferred.reject(new Error(err)); //reject it in the callback
      } else {
        ...
        choiceData['choice'] = choice;
        choiceData['score'] = score;
        deferred.resolve(choiceData); //resolve it in the callback
      }
      console.log("");
    });
  return deferred.promise; //return the promise object
};

The search function. Note the promise chain:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
exports.search = function(req, res) {
  var choices = JSON.parse(req.body.choices),
      choiceArray = [];

  var promise = function(choices) {
    var deferred = Q.defer();
    choices.forEach(function(choice, index) {
      searchTweets(choice)
        .fail(function(error) {
          throw new Error(error);
        })
        .done(function(data) {
          choiceArray.push(data);
          if (choiceArray.length === choices.length) {
            deferred.resolve(choiceArray);
          }
        });
    });
    return deferred.promise;
  };

  promise(choices).then(function(data) {
      return scoreCompare(data);
    }).done(function(result) {
      console.log('final_result', result);
      res.send(result);
    });

};

Have Fun!

Generators

Generators are the new kid on the block, but they look the most promising. Essentially, they make it easy to suspend/pause a function then resume it with the yield function.

Make sure you are using a browser that supports ES6: http://kangax.github.io/es5-compat-table/es6/#Generators_(yield)). I personally use Chrome Canary, with experimental Javasctipt enabled: “chrome://flags/#enable-javascript-harmony”.

… also …

As of Node v0.11.3, you must use the --harmony_generators flag for running applications that contain generator examples in order to enable ES6 experimental features - e.g., node --harmony_generators app.js.

Let’s look at a quick example.

Example

Open the Javascript console, then enter this generator function:

1
2
3
4
5
6
function* naturalNumbers(){
  var n = 1;
  while (true){
    yield n++;
  }
}

Next, you can call the function with this line:

1
var numbers = naturalNumbers();

Finally, you can generate an object with the returned values by calling numbers.next()

es6-generators

So, how do we add this to our Sentiment project? I’m not sure. :)

IcedCoffeeScript

Example

1
2
3
4
5
6
7
8
et = require 'errTo'
{get} = require 'request'
fn = (done) ->
  await get 'http://foo.com', et done, defer resp, body
  await get 'http://bar.com', et done, defer resp, body
  do done
await fn defer err
throw err if err

So, how do we ad this to our Sentiment project? I’m not sure. :)

Data Binding

This isn’t a method of handling the non-blocking function issue, but it instead shows how easily update the front end. We are using Async again to address the function issue. Check out the code here.

Thanks to Aaron Vandrey for developing the code and writing the following explanation.

Although there are a number of front-end MV* frameworks that could be used, we chose the KnockoutJS data binding library for simplicity. KnockoutJS uses “observables” to enable two-way data binding from the View (HTML) back to the View-model (JavaScript).

From [10 things to know about KnockoutJS on day one])http://www.knockmeout.net/2011/06/10-things-to-know-about-knockoutjs-on.html)”:

Observables are functions. The actual value and subscribers to the observable are cached internally by the function. You set an observable’s value by passing the new value as the only argument to the function and you read the value by passing no arguments.

We can use these functions to read the values from the form directly, hide and expose DIVs and change text on the screen.

From the KnockoutJS data-binding page:

Knockout’s declarative binding system provides a concise and powerful way to link data to the UI. It’s generally easy and obvious to bind to simple data properties or to use a single binding. … A binding consists of two items, the binding name and value, separated by a colon.

Server Side Code

Views

Combining the functions in our main.js (more on this later), on the client side, with Knockout’s declarative data-binding syntax, we can set up the Jade template in the manner shown below.

In the original Jade template there are placeholder DIVs set up that we then use jQuery to interact with - to display the error messages and results. We also used jQuery to update the styles applied to the DIVs. Since we are using data binding in this example, we will go ahead and set up the DIVs for errors and results and have their HTML and styles in the DOM at all times. Then using the “visible” data binding on the DIVs we can hide and expose them as needed. In the example below we have a couple of data-bind attributes that KnockoutJS will use to handle the two-way communication from the View to the ViewModel and vise-versa.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
.form-container
  form(action='', method='post', data-bind='submit: formSubmit')
    input#choice1.choice(type='text', placeholder='Choice #1...', name='choice1', data-bind='value: inputOne')
    input#choice2.choice(type='text', placeholder='Choice #2...', name='choice2', data-bind='value: inputTwo')
    input#decision.btn.btn-success.btn-lg(type='submit', value='Submit' data-bind='enable: !hasResults()')
.decision-container
  p(class='alert alert-danger' data-bind='visible: error, text: error')
  div(class='progress progress-striped active' data-bind='visible: isProcessing()')
    div(class='progress-bar progress-bar-primary' role='progressbar' aria-valuenow='100' aria-valuemin='0' aria-valuemax='100' style='width: 100%')
      span(class='sr-only')
  div(class='panel panel-lg panel-success' data-bind='visible: hasResults()')
    div(class='panel-heading')
      h3(class='panel-title') Decision Results
    div(class='panel-body')
      p(class='decision-text', data-bind='html: results')
      div(class='text-center')
        input#decision.btn.btn-success.btn-sm.text-center(type='button', value='Again?' data-bind='click: tryAgain')

In the highlighted text we can see just a few of the many data-binding possibilities.

The submit binding will handle both the “click” event of the submit button as well as a user hitting the “enter” key. In the background KnockoutJS will also perform a “preventDefault” so that the form does not attempt to submit the form to the server.

The value binding will update the ViewModel with the values entered into the text boxes. A form submit is not needed to consume these values, though in this case we are using a form submit. Alternatively we could use KnockoutJS to subscribe to the change event for these form values and begin our processing when our inputs passed validation.

The text binding will both display values in the View propagated from the ViewModel, as well and send values from the View back to the ViewModel.

The enable binding will disable the submit button when the ViewModel reports back to the View that it has results back from the Twitter Sentiment Analysis.

Client Side Code

Client Side Javascript (main.js)

The biggest difference to /public/javascripts/main.js is to create a ViewModel, and at the ViewModels closure, call KnockoutJS’s applyBindings method to enable all the two-way data binding goodness.

1
2
3
4
5
6
function ViewModel() {

    

}
ko.applyBindings(new ViewModel());

In order to grab the two choices from the form we write a small method that will take use the KnockoutJS observable’s ‘no parameter’ signature to return the values.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
self.formSubmit = function(){
    // some error handling
    if(!self.inputOne() || !self.inputTwo()){
        self.error(getError('requiredInputsError'));
    } else if(self.inputOne() === self.inputTwo()) {
        self.error(getError('sameInputError'));
    } else {
        choices.push(self.inputOne());
        choices.push(self.inputTwo());
        getDecision();
        self.error('');
        self.isProcessing(true);
    }
};

The error handling will remain the same, however in the data-binding example we set the value of our error() observable. The act of setting the value of the error observable causes it to change from being a “falsey” value to being a “truthy” value, which cause the visible data binding to also change from visible = false to visible = true. This changes the visibility of the DIV formatted for error reporting as well as set the text of the specific error we encountered.

1
p(class='alert alert-danger' data-bind='visible: error, text: error')

If no errors are encountered on subsequent submissions we can set up the array we need in the call to Twitter. We also blank out the error() observable that will hide the error reporting DIV and also set the isProcessing() observable to true which will expose the “processing” animation.

We finish up processing the results. This logic to this is essentially unchanged, however, it is shown here to further exemplify how values are set and retrieved in KnockoutJS.

1
2
3
4
5
6
7
8
9
10
function getDecision(){
    $.post('/search', { 'choices': JSON.stringify(choices) }, function(data) {
        choices.length = 0;
        var results = JSON.parse(data);

        self.results(RESULTS_START_HTML + results.choice + RESULTS_END_HTML + results.score);
        self.hasResults(true);
        self.isProcessing(false);
    });
}

The logic required to turn off the “processing” animation, expose the DIV formatted to successful results, and display the results are achieved by manipulating more observables. The isProcssing() observable is set to false to hide the animation, the hasResults() observable is set to true to expose the results DIV and finally, by setting the results() observable to some friendly copy we let the user know the outcome of the sentiment analysis. When writing this value out the page we use the html binding rather than the text binding so that we can inject HTML into the copy we are writing to the screen. If the text binding had been used, rather than the html binding, the HTML would have been encoded and we would have had the literal string <strong> written to the screen - which obviously is not what we want in this case.

main.js:

1
2
self.RESULTS_START_HTML = 'and the winner is ... <strong>';
self.RESULTS_END_HTML = '</strong> ... with a score of ';

index.jade:

1
p(class='decision-text', data-bind='html: results')

Refactor

After submitting this code it we determined that the data-binding could have been used even better by not having an error DIV and a results DIV. By taking advantage of the css binding and a KnockoutJS computed observable (an observable that can watch multiple observables and return one value) the Bootstrap class could have easily been changed from danger to success and the title and copy changed using existing observables.

Here shouldShowMessages is a computed observable that will return true if either we have an error or if we have results, otherwise it will return false. Similarly, messageType is a computed observable that will return “error” unless we have successfully received results, at which point it will return “success”.

index.jade

1
2
3
4
5
6
7
8
div(class='panel panel-lg' data-bind='visible: shouldShowMessages, css: "panel-" + messageType()')
  div(class='panel-heading')
    h3(class='panel-title' data-bind='text: messageTitle')
  div(class='panel-body')
    p(class='decision-text', data-bind='html: results')
    p(class='text-danger', data-bind='text: error')
    div(class='text-center')
      input#decision.btn.btn-success.btn-sm.text-center(type='button', value='Again?' data-bind='visible: hasResults(), click: tryAgain')

main.js:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
self.shouldShowMessages = ko.computed(function(){
    var returnValue = false;

    if (!self.isProcessing() && (self.hasResults() || self.error() > '')) {
        returnValue = true;
    }

    return returnValue;
});
self.messageType = ko.computed(function(){
    var returnValue = 'danger';

    self.messageTitle(ERROR_TITLE);
    if (self.hasResults()) {
        returnValue = 'success';
        self.messageTitle(SUCCESS_TITLE);
    }

    return returnValue;
});

It should be noted that most data-bindings will make a call to ko.utils.unwrapObservable() behind the scenes. This allows us to make the data-bind safely on both observables and non-observables. However, if you take a look at where the messageType observable is used you will notice that we are referencing the observable as a function (with parentheses). This is because we are accessing the observable inside an expression.

Conclusion

Thanks to John Rosendahl for help with writing the intro.

Pull requests are welcomed/encouraged/needed. Enjoy!

Node Twitter Sentiment

In this tutorial we’ll be building an app to pull in real-time Tweets using a Twitter client library for Node called Twit along with NodeJS, Express, and Sentimental (for sentiment analysis).

This is for the Node-js-Denver-Boulder Meetup <3 Cheers!

You can grab the example code here.

Requirements: This tutorial starts where this intro tutorial, Getting Started with Node, ends. If you’ve never set up a Node/Express application before, please start with the intro tutorial. Thanks.

twit-decision

Project Setup

As you know, Node uses Javascript for both the client and server side. Because of this, the project structure is even more important to not only separate out different concerns (client vs server) but also for your own understanding - e.g., so you can distinguish between client and server side code.

Let’s get to it.

1. Setup basic project structure with Express

1
$ express twit-decision

2. Install dependencies for Node, Express, and Jade:

1
$ cd twit-decision && npm install

3. Your project structure should now look like this:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
.
├── app.js
├── package.json
├── public
│   ├── images
│   ├── javascripts
│   └── stylesheets
│       └── style.css
├── routes
│   ├── index.js
│   └── user.js
└── views
    ├── index.jade
    └── layout.jade

What’s going on?

  • Server side code includes app.js (app configurations, middleware, and routing), the “routes” folder (controller/business logic), and the views folder (views, templates, partials)
  • Meanwhile, client side code includes the “public” folder (images, Javascript files, and stylesheets)

4. Run the server

1
$ node app

You should see the “Welcome to Express” Text.

Server Side Code

We’ll start with the server side. Our server code will be responsible for serving up our main index page, which will display two input boxes where the end user can enter data for comparison. When the data is passed to the server, via jQuery and AJAX on the client end, the server connects to Twitter, pulls the live tweets, and processes sentiment. Finally, the server sends the results back to the client.

1. Install dependences

1
2
$ npm install twit --save
$ npm install Sentimental --save

2. Updated app.js code

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
// module dependencies
var express = require('express'),
  routes = require('./routes'),
  http = require('http'),
  path = require('path'),
  fs = require('fs');

// create express app  
var app = express();

// all environments
app.set('port', process.env.PORT || 3000);
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'jade');
app.use(express.favicon());
app.use(express.logger('dev'));
app.use(express.json());
app.use(express.urlencoded());
app.use(express.methodOverride());
app.use(app.router);
app.use(express.static(path.join(__dirname, 'public')));

// development only
if ('development' == app.get('env')) {
  app.use(express.errorHandler());
}

// routes
app.get('/', routes.index);
app.get('/ping', routes.ping);

// create server
http.createServer(app).listen(app.get('port'), function(){
  console.log('Express server listening on port ' + app.get('port'));
});

You’ve seen the majority of this code already, from the original tutorial, so I won’t go into too much detail. Plus, it’s heavily commented and, right now, it resembles a pretty standard Node/Express app.

Let’s setup our routes next.

3. Routes

1
2
3
4
5
6
7
8
9
var path = require("path");

exports.index = function(req, res){
  res.render('index', { title: "Twit-Decision"});
};

exports.ping = function(req, res){
  res.send("pong!", 200);
};

Again, pretty straightforward here. We are serving up one page, index, while the second render parameter passes the title to the view. We also added a test route, called ping, which will just display ping on the page.

Test it out. Navigate to http://localhost:3000/ping. You should see “pong!” in the top left corner.

4. Views

Update index.jade

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
doctype html
html
  head
    title= title
    meta(charset='utf-8')
    meta(name='viewport', content='width=device-width, initial-scale=1.0')
    meta(name='description', content='')
    meta(name='author', content='Michael Herman')
    link(href='http://netdna.bootstrapcdn.com/bootswatch/3.1.0/yeti/bootstrap.min.css', rel='stylesheet', media='screen')
    link(href='/stylesheets/main.css', rel='stylesheet', media='screen')
  body
    .container
      .jumbotron
        h1 Need to make a decision?
        p.lead Use Twitter sentiment analysis.
        br
        br
        .form-container
          form(action='', method='post')
            input#choice1.choice(type='text', data-choice='1', placeholder='Choice #1...', name='choice1')
            input#choice2.choice(type='text', data-choice='2', placeholder='Choice #2...', name='choice2')
            input#decision.btn.btn-success.btn-lg(type='submit', value='Decide!')
        br
        br
        .decision-container
          p#status
          p#decision-text
          p#score
          input#again.btn.btn-success.btn-lg(value='Again?')
    script(src='http://code.jquery.com/jquery-1.11.0.min.js')
    script(src='http://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js')
    script(src='javascripts/main.js')

This is our only template that we need. It’s the index page, used for markup, and coded using the Jade Template Language. If this is confusing, I suggest converting this code to HTML and comparing the differences.We have the typical meta tags a links to CSS sheets in the <head>. The <body> includes a form as well as a number of selectors for appending the results of the sentiment analysis. Most of the styling is done in Bootstrap.

Let’s quickly jump to the client side.

Client Side

1. Styles

Add these custom styles to the main.css style within the “stylesheets” folder:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
.container {
  max-width: 1000px;
  padding-top: 50px;
  text-align: center;
}
.choice {
  width:100%;
  height:50px;
  font-size:25px;
  padding:10px;
}
#decision-text {
  font-weight:bold;
  font-size:60px;
}
#decision {
  margin-top:10px;
}
#status, #score {
  font-size:25px;
}
.form-container {
  margin: auto;
  max-width: 500px;
}
.decision-container {
  margin: auto;
  max-width: 500px;
}

If you’re curious, see how these CSS styles (values and properties) align up to the selectors in the jade template.

2. Client Side Javascript

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
$(function () {

  // highest # of choices (inputs) allowed
  window.highestChoice = 2;
  // hide again button on page load
  $("#again").hide();

  var goDecide = function(e) {
    // prevent default browser behavior upon submit
    e.preventDefault();
    // erase old values
    $("#status").text('');
    $("#score").text('');
    // hide decision text
    $("#decision-text").hide();
    $("#again").hide();
    // display processing text, update color to black in case of an error
    $("#status").css("color", "black");
    $("#status").text("Processing ...");
    // create variable to see if any of the inputs are input
    var anyEmpty = false;
    // array to hold inputs
    var choices = [];
    // grab values, add to choices array
    for(var i = 1; i <= window.highestChoice; i++) {
      var choiceValue = $("#choice"+i).val();
      if(choiceValue == '') {
        anyEmpty = true;
      } else {
        if(choices.indexOf(choiceValue) == -1) {
          choices.push(choiceValue);
        }
      }
    }
    // Handling *some* errors
    if(!anyEmpty) {
      if($("#choice1").val() != $("#choice2").val()) {
        // send values to server side for processing, wait for callback, getting AJAXy
        $.post('/search', {'choices': JSON.stringify(choices)}, function(data) {
          data = JSON.parse(data);
          // append data to the DOM
          $(".form-container").hide()
          $("#status").text("and the winner is ...");
          $("#decision-text").text(data['choice']);
          $("#score").text('... with a score of ' + data['score'] + '');
          $("#decision-text").fadeIn();
          $("#score").fadeIn();
          $("#again").show()
        });
      } else {
        // error code
        $("#status").css("color", "red");
        $("#status").text("Both choices are the same. Try again.");
      }
    } else {
      // error code
      $("#status").css("color", "red");
      $("#status").text("You must enter a value for both choices.");
    }
  }



  // ----- MAIN ----- //

  // on click, run the goDecide function
  $("#decision").click(goDecide);
  // on click new form is shown
  $("#again").click(function() {
    $(".form-container").show()
    $("#again").hide()
    // erase old values
    $("#status").text('');
    $("#score").text('');
    $("#choice1").val('');
    $("#choice2").val('');
    // hide decision text
    $("#decision-text").hide();
  });

});

Now comes the fun part! Add a main.js file to your “javascripts” folder.

Yes, there’s a lot going on here. Fortunately, it’s well documented.

Start with the // ----- MAIN ----- // code. This essentially controls everything else. Nothing happens until the decision button is clicked. Once that happens the goDecide() function fires. This is where things get, well, interested.

Go through it line by line, reading the comment, then code. Make sure you understand what each statement is doing.

Notice how the magic starts happening when the data is grabbed from the inputs, added to an array, and then sent to the server side via AJAX. Notice the /search endpoint. We pass the stringified choice array to that endpoint, which needs to be setup on the server side, then what for the data to comeback before appending it to the DOM.

Check out the rest of the code on your own. Follow the comments for assistance.

Back to the Server Side

So, we need to set up a new route, ‘/search’, on the server side to handle the data sent from the client side.

1. app.js

First, add the route to app.js:

1
app.post('/search', routes.search)

2. Update routes:

Then add the following code to index.js in the “routes” folder:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
exports.search = function(req, res) {
  // grab the request from the client
  var choices = JSON.parse(req.body.choices);
  // grab the current date
  var today = new Date();
  // establish the twitter config (grab your keys at dev.twitter.com)
  var twitter = new twit({
    consumer_key: config.consumer_key,
    consumer_secret: config.consumer_secret,
    access_token: config.access_token,
    access_token_secret: config.access_token_secret
  });
  // set highest score
  var highestScore = -Infinity;
  // set highest choice
  var highestChoice = null;
  // create new array
  var array = [];
  // set score
  var score = 0;
  console.log("----------")

  // iterate through the choices array from the request
  for(var i = 0; i < choices.length; i++) {
    (function(i) {
    // add choice to new array
    array.push(choices[i])
    // grad 20 tweets from today
    twitter.get('search/tweets', {q: '' + choices[i] + ' since:' + today.getFullYear() + '-' +
      (today.getMonth() + 1) + '-' + today.getDate(), count:20}, function(err, data) {
        // perform sentiment analysis
        score = performAnalysis(data['statuses']);
        console.log("score:", score)
        console.log("choice:", choices[i])
        //  determine winner
        if(score > highestScore) {
          highestScore = score;
          highestChoice = choices[i];
          console.log("winner:",choices[i])
        }
        console.log("")
      });
    })(i)
  }
  // send response back to the server side; why the need for the timeout?
  setTimeout(function() { res.end(JSON.stringify({'score': highestScore, 'choice': highestChoice})) }, 5000);  
};

Again, I’ve commented this heavily. So go through, line by line, and see what’s happening.

Points of note:

  1. Add your Twitter config keys to a new file called config.js. More on this in the next section.
  2. Why are we using a timeout? Try removing it. What happens? Why is this a bad practice?

3. Config

Open the config_example.js file. Save the file as config.js, then add your own Twitter keys. Add this as a dependency along with Twit and Sentimental to your index.js file:

1
2
3
var twit = require('twit');
var sentimental = require('Sentimental');
var config = require("./config")

4. Twitter

Remember this line from your routes file, index.js:

1
score = performAnalysis(data['statuses']);

Well, we pass the pulled tweets as arguments into the performAnalysis() function. performAnalysis Let’s add that function:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
function performAnalysis(tweetSet) {
  //set a results variable
  var results = 0;
  // iterate through the tweets, pulling the text, retweet count, and favorite count
  for(var i = 0; i < tweetSet.length; i++) {
    tweet = tweetSet[i]['text'];
    retweets = tweetSet[i]['retweet_count'];
    favorites = tweetSet[i]['favorite_count'];
    // remove the hastag from the tweet text
    tweet = tweet.replace('#', '');
    // perform sentiment on the text
    var score = sentimental.analyze(tweet)['score'];
    // calculate score
    results += score;
    if(score > 0){
      if(retweets > 0) {
        results += (Math.log(retweets)/Math.log(2));
      }
      if(favorites > 0) {
        results += (Math.log(favorites)/Math.log(2));
      }
    }
    else if(score < 0){
      if(retweets > 0) {
        results -= (Math.log(retweets)/Math.log(2));
      }
      if(favorites > 0) {
        results -= (Math.log(favorites)/Math.log(2));
      }
    }
    else {
      results += 0;
    }
  }
  // return score
  results = results / tweetSet.length;
  return results
}

After the tweets are passed in, the text is parsed and sentiment is analyzed. Finally a score is calculated and returned.

Boom. That’s it!

Your project structure should now look like:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
.
├── app.js
├── config.js
├── package.json
├── public
│   ├── images
│   ├── javascripts
│   │   └── main.js
│   └── stylesheets
│       └── main.css
├── routes
│   └── index.js
└── views
    └── index.jade

Test time!

Conclusion

Test this out a few times. Make sure it all works. Perhaps go through it iteratively, following along with the code for further understanding.

Think about what you could add to make this app more fun/unique?

  1. Perhaps add a persistence layer, such as MongoDB, to retain the history of your searches to see sentiment over time.
  2. Ability to display actual tweets.