s3_plugin.js

import http from 'http'
import https from 'https'
import fs from 'fs'
import path from 'path'
import ProgressBar from 'progress'
import cdnizer from 'cdnizer'
import _ from 'lodash'
import mime from 'mime/lite'
import {CloudFront} from '@aws-sdk/client-cloudfront'
import {S3} from '@aws-sdk/client-s3'
import {Upload} from '@aws-sdk/lib-storage'

import packageJson from '../package.json'

import {
  addSeperatorToPath,
  addTrailingS3Sep,
  getDirectoryFilesRecursive,
  testRule,
  UPLOAD_IGNORES,
  DEFAULT_UPLOAD_OPTIONS,
  REQUIRED_S3_UP_OPTS,
  PATH_SEP,
  DEFAULT_TRANSFORM,
} from './helpers'

http.globalAgent.maxSockets = https.globalAgent.maxSockets = 50

const compileError = (compilation, error) => {
  compilation.errors.push(new Error(error))
}

module.exports = class S3Plugin {
  constructor(options = {}) {
    var {
      include,
      exclude,
      progress,
      basePath,
      directory,
      htmlFiles,
      basePathTransform = DEFAULT_TRANSFORM,
      s3Options = {},
      cdnizerOptions = {},
      s3UploadOptions = {},
      cloudfrontInvalidateOptions = {},
      priority,
    } = options

    this.uploadOptions = s3UploadOptions
    this.cloudfrontInvalidateOptions = cloudfrontInvalidateOptions
    this.isConnected = false
    this.cdnizerOptions = cdnizerOptions
    this.urlMappings = []
    this.uploadTotal = 0
    this.uploadProgress = 0
    this.basePathTransform = basePathTransform
    basePath = basePath ? addTrailingS3Sep(basePath) : ''

    this.options = {
      directory,
      include,
      exclude,
      basePath,
      priority,
      htmlFiles: typeof htmlFiles === 'string' ? [htmlFiles] : htmlFiles,
      progress: _.isBoolean(progress) ? progress : true,
    }

    this.clientConfig = {
      s3Options,
      maxAsyncS3: 50,
    }

    this.noCdnizer = !Object.keys(this.cdnizerOptions).length

    if (!this.noCdnizer && !this.cdnizerOptions.files)
      this.cdnizerOptions.files = []
  }

  apply(compiler) {
    this.connect()

    const isDirectoryUpload = !!this.options.directory,
          hasRequiredUploadOpts = _.every(
            REQUIRED_S3_UP_OPTS,
            (type) => this.uploadOptions[type]
          )

    // Set directory to output dir or custom
    this.options.directory =
      this.options.directory ||
      compiler.options.output.path ||
      compiler.options.output.context ||
      '.'

    compiler.hooks.done.tapPromise(
      packageJson.name,
      async({compilation}) => {
        let error

        if (!hasRequiredUploadOpts)
          error = `S3Plugin-RequiredS3UploadOpts: ${REQUIRED_S3_UP_OPTS.join(
            ', '
          )}`

        if (error) return compileError(compilation, error)

        if (isDirectoryUpload) {
          const dPath = addSeperatorToPath(this.options.directory)

          return this.getAllFilesRecursive(dPath)
            .then((files) => this.handleFiles(files))
            .catch((e) => this.handleErrors(e, compilation))
        } else {
          return this.getAssetFiles(compilation)
            .then((files) => this.handleFiles(files))
            .catch((e) => this.handleErrors(e, compilation))
        }
      }
    )
  }

  handleFiles(files) {
    return this.changeUrls(files)
      .then((files) => this.filterAllowedFiles(files))
      .then((files) => this.uploadFiles(files))
      .then(() => this.invalidateCloudfront())
  }

  async handleErrors(error, compilation) {
    compileError(compilation, `S3Plugin: ${error}`)
    throw error
  }

  getAllFilesRecursive(fPath) {
    return getDirectoryFilesRecursive(fPath)
  }

  addPathToFiles(files, fPath) {
    return files.map((file) => ({
      name: file,
      path: path.resolve(fPath, file),
    }))
  }

  getFileName(file = '') {
    if (_.includes(file, PATH_SEP))
      return file.substring(_.lastIndexOf(file, PATH_SEP) + 1)
    else return file
  }

  getAssetFiles({assets, outputOptions}) {
    const files = _.map(assets, (value, name) => ({
      name,
      path: `${outputOptions.path}/${name}`,
    }))

    return Promise.resolve(files)
  }

  cdnizeHtml(file) {
    return new Promise((resolve, reject) => {
      fs.readFile(file.path, (err, data) => {
        if (err) return reject(err)

        fs.writeFile(file.path, this.cdnizer(data.toString()), (err) => {
          if (err) return reject(err)

          resolve(file)
        })
      })
    })
  }

  changeUrls(files = []) {
    if (this.noCdnizer) return Promise.resolve(files)

    var allHtml

    const {directory, htmlFiles = []} = this.options

    if (htmlFiles.length)
      allHtml = this.addPathToFiles(htmlFiles, directory).concat(files)
    else allHtml = files

    this.cdnizerOptions.files = allHtml.map(({name}) => `{/,}*${name}*`)
    this.cdnizer = cdnizer(this.cdnizerOptions)

    const [cdnizeFiles, otherFiles] = _(allHtml)
      .uniq('name')
      .partition((file) => /\.(html|css)/.test(file.name))
      .value()

    return Promise.all(
      cdnizeFiles.map((file) => this.cdnizeHtml(file)).concat(otherFiles)
    )
  }

  filterAllowedFiles(files) {
    return files.reduce((res, file) => {
      if (
        this.isIncludeAndNotExclude(file.name) &&
        !this.isIgnoredFile(file.name)
      )
        res.push(file)

      return res
    }, [])
  }

  isIgnoredFile(file) {
    return _.some(UPLOAD_IGNORES, (ignore) => new RegExp(ignore).test(file))
  }

  isIncludeAndNotExclude(file) {
    var isExclude,
        isInclude,
        {include, exclude} = this.options

    isInclude = include ? testRule(include, file) : true
    isExclude = exclude ? testRule(exclude, file) : false

    return isInclude && !isExclude
  }

  connect() {
    if (this.isConnected) return

    this.client = new S3(this.clientConfig.s3Options)
    this.isConnected = true
  }

  transformBasePath() {
    return Promise.resolve(this.basePathTransform(this.options.basePath))
      .then(addTrailingS3Sep)
      .then((nPath) => (this.options.basePath = nPath))
  }

  setupProgressBar(uploadFiles) {
    const progressTotal = uploadFiles.reduce((acc, {upload}) => upload.totalBytes + acc, 0)

    const progressBar = new ProgressBar('Uploading [:bar] :percent :etas', {
      complete: '>',
      incomplete: '∆',
      total: progressTotal,
    })

    var progressValue = 0

    uploadFiles.forEach(({upload}) => {
      upload.on('httpUploadProgress', ({loaded}) => {
        progressValue += loaded

        progressBar.update(progressValue / progressTotal)
      })
    })
  }

  prioritizeFiles(files) {
    const remainingFiles = [...files]
    const prioritizedFiles = this.options.priority.map((reg) =>
      _.remove(remainingFiles, (file) => reg.test(file.name))
    )

    return [remainingFiles, ...prioritizedFiles]
  }

  uploadPriorityChunk(priorityChunk) {
    const uploadFiles = priorityChunk.map((file) =>
      this.uploadFile(file.name, file.path)
    )

    return Promise.all(uploadFiles.map(({promise}) => promise))
  }

  uploadInPriorityOrder(files) {
    const priorityChunks = this.prioritizeFiles(files)
    const uploadFunctions = priorityChunks.map((priorityChunk) => () =>
      this.uploadPriorityChunk(priorityChunk)
    )

    return uploadFunctions.reduce(
      (promise, uploadFn) => promise.then(uploadFn),
      Promise.resolve()
    )
  }

  uploadFiles(files = []) {
    return this.transformBasePath().then(() => {
      if (this.options.priority) {
        return this.uploadInPriorityOrder(files)
      } else {
        const uploadFiles = files.map((file) =>
          this.uploadFile(file.name, file.path)
        )

        if (this.options.progress) {
          this.setupProgressBar(uploadFiles)
        }

        return Promise.all(uploadFiles.map(({promise}) => promise))
      }
    })
  }

  uploadFile(fileName, file) {
    let Key = this.options.basePath + fileName
    const s3Params = _.mapValues(this.uploadOptions, (optionConfig) => {
      return _.isFunction(optionConfig) ? optionConfig(fileName, file) : optionConfig
    })

    // avoid noname folders in bucket
    if (Key[0] === '/') Key = Key.substr(1)

    if (s3Params.ContentType === undefined)
      s3Params.ContentType = mime.getType(fileName)

    const Body = fs.createReadStream(file)
    const params = _.merge({Key, Body}, DEFAULT_UPLOAD_OPTIONS, s3Params)

    const upload = new Upload({client: this.client, params})

    if (!this.noCdnizer) this.cdnizerOptions.files.push(`*${fileName}*`)

    return {upload, promise: upload.done()}
  }

  invalidateCloudfront() {
    const {clientConfig, cloudfrontInvalidateOptions} = this

    if (cloudfrontInvalidateOptions.DistributionId) {
      const {
        accessKeyId,
        secretAccessKey,
        sessionToken,
      } = clientConfig.s3Options
      const cloudfront = new CloudFront({
        accessKeyId,
        secretAccessKey,
        sessionToken,
      })

      if (!_.isArray(cloudfrontInvalidateOptions.DistributionId))
        cloudfrontInvalidateOptions.DistributionId = [
          cloudfrontInvalidateOptions.DistributionId
        ]

      const cloudfrontInvalidations = cloudfrontInvalidateOptions.DistributionId.map(
        (DistributionId) =>
          new Promise((resolve, reject) => {
            cloudfront.createInvalidation({
              DistributionId,
              InvalidationBatch: {
                CallerReference: Date.now().toString(),
                Paths: {
                  Quantity: cloudfrontInvalidateOptions.Items.length,
                  Items: cloudfrontInvalidateOptions.Items,
                },
              },
            }, (err, res) => {
              if (err) reject(err)
              else resolve(res.Id)
            })
          })
      )

      return Promise.all(cloudfrontInvalidations)
    } else {
      return Promise.resolve(null)
    }
  }
}