How to make horseman chain synchronous?

I am doing something like the code below, where I have wrapped a horseman operation in a function that I want to return a value. The problem is, the chain gets started asynchronously so the function returns undefined immediately. Later on when horseman finishes, it console logs the correct value.

How can I cause the chain to block or return the value I want from the .then() / .evaluate() section?

var Horseman = require('node-horseman');
var horseman = new Horseman();

console.log("outside val: " + request("a"));

function request(query) {
  horseman
    .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0")
    .open('http://example.com/'+query)
    .evaluate(function() {
      return $('input[name="fieldName"]').val();
    })
    .then(function(value) {
      console.log("Running then");
      horseman.close();
      if (value !== undefined) {
        console.log("inside val: " + value);
        return value;
      }
    });
}

Output:

outside val: undefined
Running then
inside val: correctValue

[stephen304]

I got it thanks, I just had to rethink how my algorithm worked. I originally intended for it to branch out recursively (I’m a little used to that model as we do it that way for almost all recursive things in my classes)

Instead I made a queue and used the single recursive-like method in links.js. It works just about as fast as I need it to.

// Grab links from Google.
var Horseman = require("node-horseman");
var horseman = new Horseman();

var alphabet = "abcdefghijklmnopqrstuvwxyz1234567890".split("");
var queue = [""];
var currentQuery = queue.pop();
var results = [];

function getLinks(){
  return horseman.evaluate( function(){
    return $('input[name="fieldname"]').val();
  });
}

function scrape(){

  return new Promise( function( resolve, reject ){
    return getLinks()
    .then(function(newResult){

      if (newResult !== null && newResult !== undefined) {
        results.push(newResult);
        console.log("result " + newResult);
        for (var i = 0; i < alphabet.length; i++) {
          var letter = alphabet[i];
          queue.push(currentQuery + letter);
        }
        queue.sort(function(a, b){
          return b.length - a.length; // ASC -> a - b; DESC -> b - a
        });
      }

      if (queue.length > 0) {
        currentQuery = queue.pop();
        console.error("Checking " + currentQuery);
        return horseman
          .open('http://example.com/'+currentQuery)
          .then(scrape);
      }
    })
    .then( resolve );
  });
}

horseman
  .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0")
  .open('http://example.com/'+currentQuery)
  .then(scrape)
  .finally(function(){
    console.log(results.length);
    horseman.close();
  });

[softe1988]

var Horseman    = require('node-horseman'),
    phantom     = require('phantomjs-prebuilt'),
    CLI         = require('clui'),
    fs          = require('fs');

const getSearchParams = require('./commandLine');

var horseman = new Horseman({
  injectJquery: true,
  injectBluebird: true,
  debugPort: process.env.PORT,
  webSecurity: false 
}); 
var result = {};

//error logging
horseman
  .on('error', function (msg, trace) {
    console.log(msg, trace);
  }).on('timeout', function (timeout, msg) {
      console.log('timeout', msg);
  }).on('resourceTimeout', function (msg) {
      console.log('resourceTimeout', msg);
  }).on('resourceError', function (msg) {
      console.log('resourceError', msg);
  }).on('loadFinished', function (msg) {
      console.log('loadFinished', msg);
  }).on('loadStarted', function (msg) {
      console.log('loadStarted', msg);
  });

function getLinks(){
  return horseman.evaluate(function(){
    // This code is executed in the browser.
    var links = [];
    $("div.g h3.r a").each(function(item){
      var link = {
        title : $(this).text(),
        url : $(this).attr("href")
      };
      links.push(JSON.stringify(link, null, 3));
    });
    return links;
  });
}

function hasNextPage(){
  return horseman.exists("#pnnext");
}

function scrape(){
    return getLinks()
    .then(function(newLinks){
      fs.appendFileSync('../path/to/file.json', newLinks + "," + "\n");
        return hasNextPage()
        .then(function(hasNext){
          if (hasNext){
            return horseman
              .click("#pnnext")
              .wait(2000)
              .then( scrape );
          } 
          fs.appendFileSync('../path/to/file.json', newLinks.slice(-1));
          return ;
        })
    }).catch(function(err) {
        console.log("Error " + err.message);
        return err;
    })
  }  

function getUserInput(){
  //synchronous execution get user input then search with values
    return horseman
      .do(getSearchParams)
      .then(function(value){
        if(value !== undefined) {
          /*return value is an object of the user inputs*/
          return horseman 
            .log(value)
            .userAgent(/*Firefox*/'*Mozilla Firefox/47.0.1 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Chrome 53.0.2785.34 Internet Explorer 11.0.28 Safari 9.1.2')
            .open('https://www.google.com')
            .type('input[name="q"]', `${value.query} ${value.city},${value.state} `)
            .log(`Searching for: ${value.query} ${value.city},${value.state}`)
            .click('[name="btnG"]')
            .keyboardEvent('keypress', 16777221)
            .waitForSelector('div.g') //'div#MSOZoneCell_WebPartWPQ3'
            //.then(status.start())
            .then(fs.appendFileSync('../path/to/file.json', "[" + "\n"))
            .then(scrape)
           // .then(status.stop())
            .log("Scraping Complete")
            .finally(function(){
              fs.appendFileSync('../path/to/file.json', "]" + "\n")
              horseman.close();
            });
        }
      }).then(function(query){
        console.log("Results populated in: ../path/to/file.json");
      })
   };    
 
//runs a google search with a specific query string
horseman
  .do(getUserInput)

[Command Line Program] utilizing synchronous execution to grab the user input first then run the search using those values using horseman (https://gist.github.com/softe1988/7dab60e5cbfdd5f860fa59d1ddb44cd9)