I'm working on a custom generator that I originally wrote with grunt-init. One difference I'm noticing is grunt-init automatically processes all the files in root as templates but with yeoman generator you have to do this using .template(). I'm familiar with how to process any individual file with .template() but is it possible to process an entire directory?
This issue is an annoying one. I came across this when I used yeoman for the first time. I think the below code snippet can help you.
this.directory('scripts', 'scripts');// script is folder name
Looking at the Yeoman generator code, there doesn't seem to be a built in way to do this. The way I solved this was to copy some of the code from the built in generator code and modify it for my needs. I determine if a file is a template based on the _ prefix convention since I want to rename the files to exclude that prefix, but you could just treat every file as a template and it would work fine. This will copy all of the files in the directory, so what I also did is to exclude the .DS_STORE files that you find by default on OSX, but since that is a specific case I didn't include that here.
require('path');
MyGenerator.prototype._processDirectory = function(source, destination) {
var root = this.isPathAbsolute(source) ? source : path.join(this.sourceRoot(), source);
var files = this.expandFiles('**', { dot: true, cwd: root });
for (var i = 0; i < files.length; i++) {
var f = files[i];
var src = path.join(root, f);
if(path.basename(f).indexOf('_') == 0){
var dest = path.join(destination, path.dirname(f), path.basename(f).replace(/^_/, ''));
this.template(src, dest);
}
else{
var dest = path.join(destination, f);
this.copy(src, dest);
}
}
};
Yeoman uses mem-fs-editor, which has support for glob patterns. However the documentation is not very clear, and you may miss that point. Here is the documentation of copyTpl, that says it accepts the same options as copy. So since copy has suppport for glob patterns, copyTpl too.
At any point on your yeoman generator you can do:
const from = 'myFolder/**.js'
const to = 'project/'
this
.fs
.copyTpl(
this.templatePath(from),
this.destinationPath(to),
this.props, {interpolate: /<%=([\s\S]+?)%>/g}
);
},
Not that, if you are using a glob pattern the destination path should be a folder.
Related
I'm currently porting parts of a legacy codebase which has more than 100 themes that each come with their own css files. Those files are full with hardcoded links to Google Fonts which need to be replaced due to GDPR.
Is there some kind of automated tool available which scans through these files, replaces the link to Google Fonts and downloads all the assets? I've found a couple of semi-automated tools online but they all require copy & paste and manual download of the files. That's okay for 2-3 fonts but not for hundreds of them. Any tips for that?
I have put some efforts to create this NodeJS script.
This script searches for all css files and extracts the woff font url. Then, it replaces it with absolute path of the downloaded file against the url it found, also downloads the file in the relevant directory which can be clearly identified in the snippet as specified with fontDownloadDirectoryPath variable.
This script can be modified and improved further but as of now, provides the required functionality at its base level.
I hope this can serve as a starting point atleast to solve the stated problem or can be used completely as a solution changes few variables, given that my assumptions of few required things to get to this solution are correct.
Please feel free to modify, accordingly like the regex pattern to match something else, adding few other font types in the pattern, adding few more code to make it more robust and generalised, etc for other possibilities.
const path = require('path');
const fs = require('fs');
const https = require("https");
// update assets/css with your css path
const directoryPath = path.join(__dirname, 'assets/css');
let fontDownloadDirectoryPath = path.join(__dirname, 'assets/fonts')
let fontDownloadDirectoryFileFullPath = path.join(__dirname, 'assets/fonts/fontsDownloadUrlList.json')
fs.readdir(directoryPath, function (err, files) {
//handling error
if (err) {
return console.log('Unable to scan directory: ' + err);
}
//listing all files using forEach
files.forEach(function (file) {
// Do whatever you want to do with the file
let file_full_path = directoryPath + "/" + file
fs.readFile(file_full_path, 'utf8', (err, content) => {
if (err) {
console.error(err);
return;
}
// console.log(content);// show the content of readed file
let found = content.match(/url\(['"]([^"']+(woff2|eot|woff|ttf)["'])+\)/gi)
console.log(file_full_path, found);
let updated_content = content
if (found) {
if (fs.existsSync(fontDownloadDirectoryFileFullPath)) {
// enter the code to execute after the folder is there.
console.log('file exists')
fs.readFile(fontDownloadDirectoryFileFullPath, 'utf8', (err, read_content) => {
let read_content_json = JSON.parse(read_content)
read_content_json.push(...found)
fs.writeFile(fontDownloadDirectoryFileFullPath, JSON.stringify(read_content_json), function () { })
})
} else {
fs.writeFile(fontDownloadDirectoryFileFullPath, JSON.stringify(found), function () { })
}
console.log(found)
found.forEach((item) => {
let fontFileUrl = item.split("'")[1]
let fontFileName = fontFileUrl.split("/")[fontFileUrl.split("/").length - 1]
console.log(fontFileUrl, fontFileName)
https.get(fontFileUrl, response => {
var body = '';
var i = 0;
response.on('data', function (chunk) {
i++;
body += chunk;
console.log('BODY Part: ' + i);
});
response.on('end', function () {
console.log(body);
fs.writeFileSync(fontDownloadDirectoryPath + "/" + fontFileName, body, { encoding: 'utf8', flag: 'w' }, (err) => { console.log(err) })
console.log('Finished');
});
});
updated_content = updated_content.replace(item, "url('" + fontDownloadDirectoryPath + "/" + fontFileName + "')")
})
} else {
updated_content = content;
}
fs.writeFileSync(file_full_path, updated_content, { encoding: 'utf8', flag: 'w' })
});
});
});
I used below css file in root/assets/css directory with styles.css name for testing the above script:
#font-face {
font-family: 'BR Firma';
src: url('https://fonts.gstatic.com/s/opensans/v29/memSYaGs126MiZpBA-UvWbX2vVnXBbObj2OVZyOOSr4dVJWUgsiH0B4taVQUwaEQbjB_mQ.woff') format('woff');
font-weight: bold;
font-style: normal;
font-display: swap;
}
without having more background on the project, directory structure, and so on, I will outline how the task could be done as of now:
Scan all the directories or URLs of the project (if you run it locally or remotely) if the fonts are not being imported from one main CSS file (could happen).
Get all the Google Fonts URLs
Download all the assets (i.e. fonts from the links, maybe some pics also, etc.)
So, although you can totally do this locally with directories, here I will explain a way to do it with the browser for brevity - and possibly convenience - with Python. I am assuming you have access to the project's URLs, ofc.
You can follow this approach to scrape the URLs you want. Pass it a list from the sitemap to go through all the URLs in sequence. Then you can filter the list you get to account only for Google Fonts, simply add the in operator (as in here) to get true or false, respectively.
substring = 'https://fonts.googleapis.com'
if substring in element:
list.append(element)
else:
#do nothing, don't add it to list
Now you should have all the URLs you are interested in. If the project has several HTML pages with different fonts, those are the ones you need to scan - or maybe just all of them to be sure. Note: it is usually useful to store the list in a file to further add this code to the previous script. More info.
with open('urls.txt', 'w') as f:
f.write(element)
To download the assets, you can use this approach. Since you have all the URLs, you should be able to.
And that's pretty much it! If you add more insight into the project structure we could complete the scripts more precisely. Also, you can quickly use Jupyter Notebook to run the scripts as you tune them. In the meantime, the open details to clarify would be:
Scan websites or files?
Only HTML files or all the projects?
What to download? The font assets only?
Python script works fine for this task?
Any IDE can do, just "search and replace in files", with the appropriate patterns.
For example: PHPStorm: Find and replace text using regular expressions. Finding all the occurrences alone is already worth something and an IDE might help with "porting parts of a legacy codebase".
I've created a task with Gulp that is supposed to:
Join multiple CSS files;
Minify + remove unnecessary CSS;
Fix paths for url() directives and others;
Generate source maps;
My current code for this is:
var gulp = require("gulp"),
concat = require("gulp-concat"),
cleanCSS = require("gulp-clean-css"),
sourcemaps = require("gulp-sourcemaps");
var styleList = [
"Resources/Include/ionicons/css/ionicons.css",
"Resources/base.css",
"Resources/extra.css",
];
gulp.task("deploy-css", function() {
gulp.src(styleList)
.pipe(sourcemaps.init())
.pipe(concat("style.min.css"))
.pipe(cleanCSS({
debug: true,
compatibility: "ie8",
keepSpecialComments : 0,
target: "Resources/",
relativeTo: "Resources/"
})
)
.pipe(sourcemaps.write())
.pipe(gulp.dest("Resources/"))
});
url() path example, taken from file Resources/Include/ionicons/css/ionicons.css:
#font-face { font-family: "Ionicons"; src: url("../fonts/ionicons.eot?v=2.0.0");
This is my current file structure:
./Resources/style.min.css // -> Final processed file
./Resources/base.css
./Resources/extras.css
./Resources/Include/ // -> Original CSS files with URL (installed via Bower)
Test folder: https://dl.dropboxusercontent.com/u/2333896/gulp-path-test.zip (install and then run with gulp deploy-css).
Almost everything works as expected, except for when CSS files include references to images or fonts using the url() option. After running the task (and style.min.css created) those references are broken - no change was made to the paths found on the original CSS files.
Isn't cleanCSS supposed to check where the referenced files are and fix the paths automatically? Aren't the options target and relativeTo used to control that?
How can I fix this? Thank you.
I managed to fix the issue, my main problems were a misplaced concat operation breaking gulp-clean-css rebase feature and wrong target and relativeTo options. Apparently I didn't think much about the previous workflow.
var gulp = require("gulp"),
concat = require("gulp-concat"),
cleanCSS = require("gulp-clean-css"),
sourcemaps = require("gulp-sourcemaps");
var styleList = [
"Resources/Include/ionicons/css/ionicons.css",
"Resources/base.css",
"Resources/extra.css",
"Resources/Include/teste/base.css"
];
gulp.task("deploy-css", function() {
gulp.src(styleList)
.pipe(sourcemaps.init())
.pipe(cleanCSS({
compatibility: "ie8",
keepSpecialComments : 0,
target: "Resources",
relativeTo: ""
})
)
.pipe(concat("style.min.css", {newLine: ""}))
.pipe(sourcemaps.write())
.pipe(gulp.dest("Resources"))
});
This new workflow works as:
Optimize all individual CSS files - including rebasing urls;
Contact individual optimized files into the final file - (note newLine: "" avoids line breaks in the file);
Write the file.
I'm using cssmin with the following "Content" folder structure:
src
|--dir1
| |--style1.css
| |--images
| |--image1.png
|--dir2
|--style2.css
|--images
|--image2.png
dist
|--styles.min.css
|--styles.min.css.map
Where styles.min.css and styles.min.css.map are the result of concatenating/minifying all stylesheets in the"src" folder.
I first had issues where styles.min.css contained URLs for images in the wrong places (i.e. "images/image1.png" instead of "../src/dir1/images/image1.png") but thankfully this grunt configuration fixed that:
cssmin: {
options: {
rebase: true,
sourceMap: true
},
all: {
options: {
keepSpecialComments: 0
},
files: {
'content/dist/styles.min.css': ["content/src/dir1/style1.css", "content/src/dir2/style2.css"]
}
}
}
The new problem: The generated sourcemap ("styles.min.css.map") contains sources like this: ["content/src/dir1/style1.css", "content/src/dir2/style2.css"] instead of ["../src/dir1/style1.css", "../src/dir2/style2.css"]. This means the map is pointing to the incorrect locations, such as:
"content/dist/content/src/dir1/style1.css" and "content/dist/content/src/dir2/style2.css"
What can I do to resolve this?
For reference, I have also tried the csswring, however despite sourcemaps working fine, I found general image/import url rebasing wasn't working properly, so went back to cssmin.
Thanks very much!
Came up with my own solution. I wrote a task which reads the source map JSON, gets the array of sources, rebases them, then writes the file again. This solution seems to work well for me, hopefully this can help someone else too if they're in a similar situation. Just run your cssmin task and then this one:
grunt.registerTask("rebase-css-sourcemap-sources", "Rebases the CSS source map urls", function() {
var filePath = "./content/dist/styles.min.css.map";
if (grunt.file.exists(filePath)) {
var sourceMap = grunt.file.readJSON(filePath);
var sources = sourceMap.sources;
if (sources) {
for (var i = 0; i < sources.length; i++) {
sources[i] = sources[i].replace("content/src", "../src");
}
grunt.file.write(filePath, JSON.stringify(sourceMap));
grunt.log.ok("Rebased CSS source map source urls.");
}
} else {
grunt.log.error("Source map file does not exist: " + filePath);
}
});
While this solution works for me, if anyone knows of an alternative method of solving this problem which ideally just uses cssmin, that would be better.
I'm toying around with ASP.NET 5 and am using gulp. I added angularjs and angular-route to my package.json file which stored the files at Dependencies->NPM. I added this to my gulpfile.js thinking that it would copy over the the correct JS files. It did copy over the files, however, it also crashed the project. I had to manually go into the lib folder and remove everything that gulp added. What's the proper way to copy files from the NPM folder a destination folder. I'd like to be able to just run the task from Task Runner.
I'm assuming this is incorrect: (which is what I ran)
gulp.task("copyJs", function () {
return gulp.src('./node_modules/**/*.js')
.pipe(gulp.dest('./wwwroot/lib/'))
});
*I think the trailing '/' in gulp.dest('./wwwroot/lib/') might be the cause of the problem, try gulp.dest('./wwwroot/lib') instead.
This is the gulp workflow I use for Angular 2 with Asp.Net 5.
var gulp = require("gulp"),
merge = require("merge-stream"),
rimraf = require("rimraf");
var paths = {
webroot: "./wwwroot/",
node_modules: "./node_modules/"
};
paths.libDest = paths.webroot + "lib/";
gulp.task("clean:libs", function (cb) {
rimraf(paths.libDest, cb);
});
gulp.task("copy:libs", ["clean:libs"], function () {
var angular2 = gulp.src(paths.node_modules + "angular2/bundles/**/*.js")
.pipe(gulp.dest(paths.libDest + "angular2"));
var es6_shim = gulp.src([
paths.node_modules + "es6-shim/*.js",
"!**/Gruntfile.js"])
.pipe(gulp.dest(paths.libDest + "es6-shim"));
var systemjs = gulp.src(paths.node_modules + "systemjs/dist/*.js")
.pipe(gulp.dest(paths.libDest + "systemjs"));
var rxjs = gulp.src(paths.node_modules + "rxjs/bundles/**/*.js")
.pipe(gulp.dest(paths.libDest + "rxjs"));
return merge(angular2, es6_shim, systemjs, rxjs);
});
There are many ways to do it but one of the good simple ways I found was this: http://www.hanselman.com/blog/ControlHowYourBowerPackagesAreInstalledWithAGulpfileInASPNET5.aspx
Which do an update to the bowerrc file and everything after this update makes more sense.
UPDATE YOUR .BOWERRC AND PROJECT.JSON
In the root of your project is a .bowerrc file. It looks like this:
> { "directory": "wwwroot/lib" } Change it to something like this, and
> delete your actual wwwroot/lib folder.
>
> { "directory": "bower_components" } EXCLUDE YOUR SOURCE BOWER FOLDER
> FROM YOUR PROJECT.JSON
You'll also want to go into your project.json file for ASP.NET 5 and
make sure that your source bower_components folder is excluded from
the project and any packing and publishing process.
> "exclude": [
> "wwwroot",
> "node_modules",
> "bower_components" ],
UPDATE YOUR GULPFILE.JS
In your gulpfile, make sure that path is present in paths. There are
totally other ways to do this, including having gulp install bower and
figure out the path. It's up to you how sophisticated you want your
gulpfile to get as long as the result is that production ready .js
ends up in your wwwroot ready to be served to the customer. Also
include a lib or destination for where your resulting JavaScript gets
copied. Could be scripts, could be js, could be lib as in my case.
var paths = {
webroot: "./" + project.webroot + "/",
bower: "./bower_components/",
lib: "./" + project.webroot + "/lib/" }; ADD A COPY TASK TO YOUR GULPFILE
Now open your Gulpfile and note all the tasks. You're going to add a
copy task to copy in just the files you want for deployment with your
web app.
Here is an example copy task:
> gulp.task("copy", ["clean"], function () {
> var bower = {
> "bootstrap": "bootstrap/dist/**/*.{js,map,css,ttf,svg,woff,eot}",
> "bootstrap-touch-carousel": "bootstrap-touch-carousel/dist/**/*.{js,css}",
> "hammer.js": "hammer.js/hammer*.{js,map}",
> "jquery": "jquery/jquery*.{js,map}",
> "jquery-validation": "jquery-validation/jquery.validate.js",
> "jquery-validation-unobtrusive": "jquery-validation-unobtrusive/jquery.validate.unobtrusive.js"
> }
>
> for (var destinationDir in bower) {
> gulp.src(paths.bower + bower[destinationDir])
> .pipe(gulp.dest(paths.lib + destinationDir));
> } });
Do note this is a very simple and very explicit copy tasks. Others
might just copy more or less, or even use a globbing wildcard.
It's up to you. The point is, if you don't like a behavior in ASP.NET
5 or in the general build flow of your web application you have more
power than ever before.
Right click the Bower node in the Solution Explorer and "Restore
Packages." You can also do this in the command line or just let it
happen at build time.
Looking in this simplified screenshot, you can see the bower
dependencies that come down into the ~/bower_components folder. Just
the parts I want are moved into the ~/wwwroot/lib/** folder when the
gulpfile runs the copy task.
I manage very complex monorepos, I don't like hardcoded file paths and prefer to mirror my source code for transparency. I explored a LOT of solutions for doing a lot of files at once and find them all opaque and bloated. I recommend a factory that ultimately does this with source-like file module references:
gulp.parallel(
() =>
gulp
.src(require.resolve('#bootstrap/core/dist/bootstrap.all.min.js'))
.pipe(gulp.dest(DIST)),
() =>
gulp
.src(require.resolve('foobar/dist/foobar.all.min.js'))
.pipe(gulp.dest(DIST))
You can make them named functions for visibility as well.
I'd like to be able to run the same builds for different targets by passing in config information from a build file, e.g. grunt build:target1 and grunt build:target2...
I figured I could access the target within the grunt file
module.exports = function ( grunt ) {
var userConfig = require( **'./'+grunt.task.current.name+'build.config.js'** );
var taskConfig...
grunt.initConfig( grunt.util._.extend( userConfig, taskConfig ) );
But the target is only available within a task.
Is there another way of accomplishing something like this?
You can pass command line arguments to Grunt by passing them using two dashes, like
--[your_arg_name]=[arg_value].
Example:
grunt --target=debug
Then retrieve this value in your Grunt config file by calling
module.exports = function (grunt) {
var target = grunt.option('target'),
userConfig = require('./' + target + "build.config.js");
...
}
You can choose whichever name you like, I chose target in my examples above.