Rails 6 [Webpacker] Compiling... takes longs time in development enviroment - ruby-on-rails

Question

Webpacker (5.1.1) with Rails 6.1.4 takes a long time while compiling css and js files for existing project and extremely very slow.
[Web packer] Compiling...
Environments: Ruby 2.7.2, Node(v14.15.5), NPM 6.14.11 and yarn 1.22.4
This happens once there is no problem, i.e. when the server starts and initiating the first request.
The problem is, it starts compiling whenever I change the code in js or css and takes again the same time around 4/5 minutes.
I think compiling everything instead of the modified file.
I also tried live reloading ./bin/webpack-dev-server, It also produced the same result.
After doing Google, I tried a few steps below but to no avail
rm -rf node_modules rake assets:clobber yarn install
Did I miss any configuration/package related to compilation. package.json file looks like:
{
"private": true,
"dependencies": {
"#babel/core": "^7.4.5",
"#babel/plugin-proposal-class-properties": "^7.4.4",
"#babel/plugin-proposal-object-rest-spread": "^7.4.4",
"#babel/plugin-proposal-optional-chaining": "^7.7.4",
"#babel/plugin-syntax-dynamic-import": "^7.2.0",
"#babel/polyfill": "^7.4.4",
"#babel/preset-env": "^7.4.5",
"#babel/preset-react": "^7.0.0",
"#rails/webpacker": "^5.1.1",
"#ttskch/select2-bootstrap4-theme": "^1.3.4",
"autosuggest-highlight": "^3.1.1",
"axios": "^0.18.0",
"babel-eslint": "^8.2.3",
"babel-loader": "^8.1.0",
"babel-plugin-module-resolver": "^3.1.1",
"babel-plugin-syntax-dynamic-import": "^6.18.0",
"bootstrap": "4.1.3",
"classnames": "^2.2.6",
"clipboard": "^2.0.6",
"date-arithmetic": "^4.0.1",
"dom-helpers": "^3.4.0",
"dotdotdot-js": "^4.0.9",
"easyzoom": "^2.5.2",
"eslint": "^4.19.1",
"eslint-config-airbnb": "^16.1.0",
"eslint-plugin-babel": "^5.1.0",
"eslint-plugin-import": "^2.12.0",
"eslint-plugin-jsx-a11y": "^6.0.3",
"eslint-plugin-react": "^7.9.1",
"expose-loader": "^0.7.5",
"extract-text-webpack-plugin": "^3.0.2",
"fast-deep-equal": "^2.0.1",
"font-awesome": "^4.7.0",
"formik": "^1.5.0",
"hard-source-webpack-plugin": "^0.13.1",
"history": "^4.9.0",
"immutability-helper": "^2.7.0",
"jquery": "^3.3.1",
"jquery-bar-rating": "^1.2.2",
"jquery-countdown": "^2.2.0",
"jquery-ui": "^1.12.1",
"libphonenumber-js": "^1.7.8",
"lodash": "^4.17.11",
"moment": "^2.22.2",
"moment-locales-webpack-plugin": "^1.0.7",
"moment-timezone": "^0.5.23",
"moment-timezone-data-webpack-plugin": "^1.0.3",
"odometer": "^0.4.8",
"pc-bootstrap4-datetimepicker": "^4.17.50",
"popper": "^1.0.1",
"postcss-cssnext": "^3.1.0",
"prop-types": "^15.6.1",
"rails-erb-loader": "^5.5.2",
"rails-ujs": "^5.2.4-2",
"rc-time-picker": "3.4.0",
"react": "^16.8.6",
"react-actioncable-provider": "^1.0.3",
"react-autosuggest": "^9.3.4",
"react-copy-to-clipboard": "^5.0.2",
"react-cropper": "^1.2.0",
"react-custom-scrollbars": "git://github.com/sivakovigor/react-custom-scrollbars.git",
"react-dates": "git://github.com/ihar-lapitski/react-dates.git#feature/change-month-on-date-change",
"react-dom": "^16.8.6",
"react-dotdotdot": "^1.2.3",
"react-dropzone": "^10.1.4",
"react-facebook-login": "^4.0.1",
"react-geosuggest": "^2.12.0",
"react-google-maps": "^9.4.5",
"react-helmet": "^5.2.1",
"react-image-magnify": "^2.7.4",
"react-lazyload": "^2.5.0",
"react-loader-spinner": "^2.2.0",
"react-overlays": "^1.2.0",
"react-paginate": "^6.0.0",
"react-places-autocomplete": "^7.2.0",
"react-player": "^1.7.1",
"react-rating": "^1.6.2",
"react-redux": "^5.0.7",
"react-rnd": "^9.0.4",
"react-router-dom": "^5.0.0",
"react-select": "^2.4.2",
"react-slick": "^0.23.1",
"react-text-mask": "^5.4.3",
"react-textarea-autosize": "^7.1.0",
"react-to-print": "^2.4.0",
"react-toastify": "^5.0.0",
"react-transition-group": "^2.5.0",
"react-validation": "^3.0.7",
"reactstrap": "^8.0.0",
"recompose": "^0.27.1",
"redux": "^4.0.0",
"redux-thunk": "^2.3.0",
"reselect": "^4.0.0",
"scroll": "^2.0.3",
"scroll-doc": "^0.2.1",
"select2": "^4.0.6-rc.1",
"slick-carousel": "^1.8.1",
"startbootstrap-sb-admin": "^4.0.0",
"turbolinks": "^5.2.0",
"uncontrollable": "^7.0.0",
"url-search-params-polyfill": "^5.0.0",
"validator": "^10.6.0",
"vanilla-lazyload": "^12.0.0",
"webp-loader": "^0.4.0",
"webpack-bundle-analyzer": "^3.8.0",
"webpack-cli": "^4.9.1",
"webpack-dev-server": "^3.11.0",
"webpack-merge": "^4.2.2",
"whatwg-fetch": "^3.0.0",
"yup": "^0.27.0"
},
"devDependencies": {
"#babel/plugin-transform-runtime": "^7.16.4"
}
}
Thanks.