Parallel piping

Hi I’m using 4.0.0-alpha1 (00cd1fdf813a6b52bcb2985aa9c3514ebb9227b6) and am wondering if there is a way to do parallel piping? Right now, inject injects script/stylesheet tags sequentially. First it’ll do css, and then it’ll do js. However, css and js have no relationship! It doesn’t make sense to wait for the tasks to finish sequentially.

function js(prod)
{
    return gulp.src(bowerSrc.ext('js').files.concat(require('./tasks/pipeline').jsFilesToInject))
        .pipe(prod ? concat('yebrithang.js') : noop())
        .pipe(prod ? uglify() : noop())
        .pipe(gulp.dest('.tmp/public/js'));
}

function css(prod)
{
    return gulp.src('assets/styles/**/*.less')
        .pipe(less())
        .pipe(addSrc.prepend(bowerSrc.ext('css').files.concat(require('./tasks/pipeline').cssFilesToInject)))
        .pipe(autoprefixer())
        .pipe(prod ? concat('yebrithang.css') : noop())
        .pipe(prod ? minifyCss({keepSpecialComments: 0}) : noop())
        .pipe(gulp.dest('.tmp/public/css'));
}

function injec(prod)
{
    return function()
    {
        return gulp.src('views/layout.ejs')
            .pipe(inject(css(prod), {ignorePath: '.tmp/public'}))
            .pipe(inject(js(prod), {ignorePath: '.tmp/public'}))
            .pipe(gulp.dest('views'));
    }
}

I’ve come close to doing it in parallel by using bach.parallel and then merge-streaming the result, but the problem is since I’m not returning a stream, the task will never be complete. I will get this output

[00:32:03] Starting 'default'...
[00:32:03] Starting 'clean'...
[00:32:03] Finished 'clean' after 34 ms
[00:32:03] Starting 'parallel'...
[00:32:03] Starting 'bowerFonts'...
[00:32:03] Starting '<anonymous>'...
[00:32:03] Starting 'images'...
starting css
starting js
[00:32:03] Finished 'images' after 538 ms
[00:32:03] Finished 'bowerFonts' after 544 ms
[00:32:03] gulp-inject 18 files into layout.ejs.

for

function js(prod)
{
    return function(cb){
        console.log('starting js')
    return cb(null, gulp.src(bowerSrc.ext('js').files.concat(require('./tasks/pipeline').jsFilesToInject))
        .pipe(prod ? concat('yebrithang.js') : noop())
        .pipe(prod ? uglify() : noop())
        .pipe(gulp.dest('.tmp/public/js')));
    }
}

function css(prod)
{
    return function(cb){
        console.log('starting css')
    return cb(null, gulp.src('assets/styles/**/*.less')
        .pipe(less())
        .pipe(addSrc.prepend(bowerSrc.ext('css').files.concat(require('./tasks/pipeline').cssFilesToInject)))
        .pipe(autoprefixer())
        .pipe(prod ? concat('yebrithang.css') : noop())
        .pipe(prod ? minifyCss({keepSpecialComments: 0}) : noop())
        .pipe(gulp.dest('.tmp/public/css')));
    }
}

function injec(prod)
{
    return function()
    {
        return bach.parallel(css(prod), js(prod))(function(err, res)
        {
        return gulp.src('views/layout.ejs')
            .pipe(inject(merge(res), {ignorePath: '.tmp/public'}))
            //.pipe(inject(js(prod), {ignorePath: '.tmp/public'}))
            .pipe(gulp.dest('views'));
        })

    }
}

here’s the default task

gulp.set
(
    'default',
    gulp.series
    (
        clean,
        gulp.parallel
        (
            bowerFonts,
            injec(false),
            images
        ),
        wat
    )
);

Now you’re probably wondering why I can’t just do

gulp.set
(
    'default',
    gulp.series
    (
        clean,
        gulp.parallel
        (
            bowerFonts,
            css(false),
            js(false),
            images
        ),
        injec,
        wat
    )
);

I could, but then I would have to glob the files again and somehow enforce an order. The css task and the js task have already done the work so why duplicate it?

[qtiki]

Okay so it seems that my problem was that I was using through2 for the transform. The parallel transform can be achieved with the aptly named parallel-transform package, which even supports max concurrency option. So by changing the gulpfile to this:

import * as gulp from 'gulp';
import transform from 'parallel-transform';
import log from 'fancy-log';

gulp.task('test', () => gulp.src('temp/**/*')
    .pipe(transform(10, (file, callback) => {
        log(`Start transform '${file.relative}'`);
        setTimeout(() => {
            log(`End transform '${file.relative}: ${file.contents.toString()}`);
            callback(null, file);
        }, 3000);
    }))
);

We get the following output:

[18:03:19] Starting 'test'...
[18:03:19] Start transform 'file1.txt'
[18:03:19] Start transform 'file2.txt'
[18:03:19] Start transform 'file3.txt'
[18:03:22] End transform 'file1.txt: Hello world from file1!
[18:03:22] End transform 'file2.txt: Hello world from file2!
[18:03:22] End transform 'file3.txt: Hello world from file3!
[18:03:22] Finished 'test' after 3.02 s

So in short: nothing wrong with gulp.src(...).

[qtiki]

This is a very old issue but I just ran into this. I have been under the impression that gulp.src(...) would do parallel piping but it doesn’t seem like so.

Here’s a quick test case with gulp version 4.0.2:

I have a dir called temp with 3 files, file1.txt, file2.txt and file3.txt. My gulpfile looks like this:

import * as gulp from 'gulp';
import { obj as through } from 'through2';
import log from 'fancy-log';

gulp.task('test', () => gulp.src('temp/**/*')
    .pipe(through((file, encoding, callback) => {
        log(`Start transform '${file.relative}'`);
        setTimeout(() => {
          log(`End transform '${file.relative}: ${file.contents.toString()}`);
          callback(null, file);
        }, 3000);
    }))
);

The output from the gulp task:

[17:15:04] Starting 'test'...
[17:15:04] Start transform 'file1.txt'
[17:15:07] End transform 'file1.txt: Hello world from file1!
[17:15:07] Start transform 'file2.txt'
[17:15:10] End transform 'file2.txt: Hello world from file2!
[17:15:10] Start transform 'file3.txt'
[17:15:13] End transform 'file3.txt: Hello world from file3!
[17:15:13] Finished 'test' after 9.02 s

I would expect all of the “Start transform” logs to happen instantly and then all the “End transform” logs after 3 seconds. However it takes 3 second for each of the transforms to complete sequentially before starting the next one.