Add s3_key_prefix

README.md

@@ -323,6 +323,14 @@ redirects:
   music-files/promo.mp4: http://www.youtube.com/watch?v=dQw4w9WgXcQ
 ```
 
+On terminology: the left value is the redirect source and the right value is the redirect
+target. For example above, *about.php* is the redirect source and */about.html* the target.
+
+If the `s3_key_prefix` setting is defined, it will be applied to the redirect
+target if and only if the redirect target points to a site-local resource and
+does not start with a slash. E.g., `about.php: about.html` will be translated
+into `about.php: VALUE-OF-S3_KEY_PREFIX/about.html`.
+
 #### Routing Rules
 
 You can configure more complex redirect rules by adding the following
@@ -396,6 +404,17 @@ operation would actually do if run without the dry switch.
 You can use the dry run mode if you are unsure what kind of effects the `push`
 operation would cause to your live website.
 
+### S3 website in a subdirectory of the bucket
+
+If your S3 website shares the same S3 bucket with other applications, you can
+push your website into a "subdirectory" on the bucket.
+
+Define the subdirectory like so:
+
+```yaml
+s3_key_prefix: your-subdirectory
+```
+
 ## Migrating from v1 to v2
 
 Please read the [release note](/changelog.md#200) on version 2. It contains

changelog.md

@@ -2,6 +2,10 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## NEXT
+
+* Add the `s3_key_prefix` setting
+
 ## 2.10.0
 
 * Support glob hashes in `cache_control`

src/main/scala/s3/website/CloudFront.scala

@@ -81,7 +81,7 @@ object CloudFront {
       if (containsPotentialDefaultRootObject) Some("/") else None
     }
     val indexPath = config.cloudfront_invalidate_root collect {
-      case true if pushSuccessReports.nonEmpty => "/index.html"
+      case true if pushSuccessReports.nonEmpty => config.s3_key_prefix.map(prefix => s"/$prefix").getOrElse("") + "/index.html"
     }
 
     val invalidationPaths: Seq[String] = {

src/main/scala/s3/website/S3.scala

@@ -116,21 +116,28 @@ object S3 {
   def awsS3Client(config: Config) = new AmazonS3Client(awsCredentials(config))
 
   def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil,  attempt: Attempt = 1)
-                              (implicit config: Config, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushOptions: PushOptions):
+                              (implicit site: Site, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushOptions: PushOptions):
   Future[Either[ErrorReport, Seq[S3File]]] = Future {
     logger.debug(nextMarker.fold
       ("Querying S3 files")
       {m => s"Querying more S3 files (starting from $m)"}
     )
-    val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
+    val objects: ObjectListing = s3Settings.s3Client(site.config).listObjects({
       val req = new ListObjectsRequest()
-      req.setBucketName(config.s3_bucket)
+      req.setBucketName(site.config.s3_bucket)
       nextMarker.foreach(req.setMarker)
       req
     })
     objects
   } flatMap { (objects: ObjectListing) =>
-    val s3Files = alreadyResolved ++ (objects.getObjectSummaries.toIndexedSeq.toSeq map (S3File(_)))
+
+    /**
+     * We could filter the keys by prefix already on S3, but unfortunately s3_website test infrastructure does not currently support testing of that.
+     * Hence fetch all the keys from S3 and then filter by s3_key_prefix.
+     */
+    def matchesPrefix(os: S3ObjectSummary) = site.config.s3_key_prefix.fold(true)(prefix => os.getKey.startsWith(prefix))
+
+    val s3Files = alreadyResolved ++ (objects.getObjectSummaries.filter(matchesPrefix).toIndexedSeq.toSeq map (S3File(_)))
     Option(objects.getNextMarker)
       .fold(Future(Right(s3Files)): Future[Either[ErrorReport, Seq[S3File]]]) // We've received all the S3 keys from the bucket
       { nextMarker => // There are more S3 keys on the bucket. Fetch them.

src/main/scala/s3/website/UploadHelper.scala

@@ -1,5 +1,6 @@
 package s3.website
 
+import s3.website.S3Key.isIgnoredBecauseOfPrefix
 import s3.website.model.Files.listSiteFiles
 import s3.website.model._
 import s3.website.Ruby.rubyRegexMatches
@@ -47,7 +48,7 @@ object UploadHelper {
   def resolveDeletes(s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
                     (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] =
     if (site.config.ignore_on_server exists (
-      ignoreRegexes => ignoreRegexes.s3KeyRegexes exists( regex => regex matches S3Key(DELETE_NOTHING_MAGIC_WORD))
+      ignoreRegexes => ignoreRegexes.s3KeyRegexes exists( regex => regex matches S3Key.build(DELETE_NOTHING_MAGIC_WORD, site.config.s3_key_prefix))
     )) {
       logger.debug(s"Ignoring all files on the bucket, since the setting $DELETE_NOTHING_MAGIC_WORD is on.")
       Future(Right(Nil))
@@ -58,7 +59,10 @@ object UploadHelper {
         for {
           remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right
         } yield {
-          val keysToRetain = (localS3Keys ++ (redirects map { _.s3Key })).toSet
+          val keysIgnoredBecauseOf_s3_key_prefix = remoteS3Keys.filterNot(isIgnoredBecauseOfPrefix)
+          val keysToRetain = (
+            localS3Keys ++ (redirects map { _.s3Key }) ++ keysIgnoredBecauseOf_s3_key_prefix
+          ).toSet
           remoteS3Keys filterNot { s3Key =>
             val ignoreOnServer = site.config.ignore_on_server.exists(_ matches s3Key)
             if (ignoreOnServer) logger.debug(s"Ignoring $s3Key on server")

src/main/scala/s3/website/model/Config.scala

@@ -20,6 +20,7 @@ case class Config(
   cache_control:              Option[Either[String, S3KeyGlob[String]]],
   gzip:                       Option[Either[Boolean, Seq[String]]],
   gzip_zopfli:                Option[Boolean],
+  s3_key_prefix:              Option[String],
   ignore_on_server:           Option[S3KeyRegexes],
   exclude_from_upload:        Option[S3KeyRegexes],
   s3_reduced_redundancy:      Option[Boolean],
@@ -119,14 +120,14 @@ object Config {
       }
     }
 
-  def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[S3Key, String]]] = {
+  def loadRedirects(s3_key_prefix: Option[String])(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[S3Key, String]]] = {
     val key = "redirects"
     val yamlValue = for {
       redirectsOption <- loadOptionalValue(key)
       redirectsOption <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
     } yield Right(redirectsOption.map(
         redirects => redirects.map(
-          ((key: String, value: String) => (S3Key(key), value)).tupled
+          ((key: String, value: String) => (S3Key.build(key, s3_key_prefix), value)).tupled
         )
       ))
 

src/main/scala/s3/website/model/Site.scala

@@ -15,7 +15,10 @@ import s3.website.model.Config.UnsafeYaml
 import scala.util.Success
 
 case class Site(rootDirectory: File, config: Config) {
-  def resolveS3Key(file: File) = S3Key(file.getAbsolutePath.replace(rootDirectory.getAbsolutePath, "").replace(File.separator,"/").replaceFirst("^/", ""))
+  def resolveS3Key(file: File) = S3Key.build(
+    file.getAbsolutePath.replace(rootDirectory.getAbsolutePath, "").replace(File.separator,"/").replaceFirst("^/", ""),
+    config.s3_key_prefix
+  )
 }
 
 object Site {
@@ -40,13 +43,14 @@ object Site {
           gzip <- loadOptionalBooleanOrStringSeq("gzip").right
           gzip_zopfli <- loadOptionalBoolean("gzip_zopfli").right
           extensionless_mime_type <- loadOptionalString("extensionless_mime_type").right
+          s3_key_prefix <- loadOptionalString("s3_key_prefix").right
           ignore_on_server <- loadOptionalS3KeyRegexes("ignore_on_server").right
           exclude_from_upload <- loadOptionalS3KeyRegexes("exclude_from_upload").right
           s3_reduced_redundancy <- loadOptionalBoolean("s3_reduced_redundancy").right
           cloudfront_distribution_id <- loadOptionalString("cloudfront_distribution_id").right
           cloudfront_invalidate_root <- loadOptionalBoolean("cloudfront_invalidate_root").right
           concurrency_level <- loadOptionalInt("concurrency_level").right
-          redirects <- loadRedirects.right
+          redirects <- loadRedirects(s3_key_prefix).right
           treat_zero_length_objects_as_redirects <- loadOptionalBoolean("treat_zero_length_objects_as_redirects").right
         } yield {
           gzip_zopfli.foreach(_ => logger.info(
@@ -66,6 +70,7 @@ object Site {
             cache_control,
             gzip,
             gzip_zopfli,
+            s3_key_prefix,
             ignore_on_server = ignore_on_server,
             exclude_from_upload = exclude_from_upload,
             s3_reduced_redundancy,

src/main/scala/s3/website/model/push.scala

@@ -132,7 +132,7 @@ object Files {
       val excludeByConfig = site.config.exclude_from_upload exists {
         _.s3KeyRegexes.exists(_ matches s3Key)
       }
-      val neverUpload = "s3_website.yml" :: ".env" :: Nil map (k => S3Key(k))
+      val neverUpload = "s3_website.yml" :: ".env" :: Nil map (k => S3Key.build(k, site.config.s3_key_prefix))
       val doNotUpload = excludeByConfig || (neverUpload contains s3Key)
       if (doNotUpload) logger.debug(s"Excluded $s3Key from upload")
       doNotUpload
@@ -157,7 +157,7 @@ object Redirect {
     val redirectSettings = config.redirects.fold(Nil: Seq[RedirectSetting]) { sourcesToTargets =>
       sourcesToTargets.foldLeft(Seq(): Seq[RedirectSetting]) {
         (redirects, sourceToTarget) =>
-          redirects :+ RedirectSetting(sourceToTarget._1, applySlashIfNeeded(sourceToTarget._2))
+          redirects :+ RedirectSetting(sourceToTarget._1, applyRedirectRules(sourceToTarget._2))
       }
     }
     def redirectsWithExistsOnS3Info =
@@ -178,13 +178,13 @@ object Redirect {
       allConfiguredRedirects
   }
 
-  private def applySlashIfNeeded(redirectTarget: String) = {
+  private def applyRedirectRules(redirectTarget: String)(implicit config: Config) = {
     val isExternalRedirect = redirectTarget.matches("https?:\\/\\/.*")
     val isInSiteRedirect = redirectTarget.startsWith("/")
     if (isInSiteRedirect || isExternalRedirect)
       redirectTarget
     else
-      "/" + redirectTarget // let the user have redirect settings like "index.php: index.html" in s3_website.ml
+      s"${config.s3_key_prefix.map(prefix => s"/$prefix").getOrElse("")}/$redirectTarget"
   }
 
   def apply(redirectSetting: RedirectSetting, needsUpload: Boolean): Redirect =
@@ -194,5 +194,6 @@ object Redirect {
 case class S3File(s3Key: S3Key, md5: MD5, size: Long)
 
 object S3File {
-  def apply(summary: S3ObjectSummary): S3File = S3File(S3Key(summary.getKey), summary.getETag, summary.getSize)
+  def apply(summary: S3ObjectSummary)(implicit site: Site): S3File =
+    S3File(S3Key.build(summary.getKey, None), summary.getETag, summary.getSize)
 }

src/main/scala/s3/website/package.scala

@@ -60,9 +60,19 @@ package object website {
     def matches(s3Key: S3Key) = rubyRegexMatches(s3Key.key, keyRegex.pattern.pattern())
   }
 
-  case class S3Key(key: String) {
+  trait S3Key {
+    val key: String
     override def toString = key
   }
+
+  object S3Key {
+    def prefix(s3_key_prefix: Option[String]) = s3_key_prefix.map(prefix => if (prefix.endsWith("/")) prefix else prefix + "/").getOrElse("")
+
+    def isIgnoredBecauseOfPrefix(s3Key: S3Key)(implicit site: Site) = s3Key.key.startsWith(prefix(site.config.s3_key_prefix))
+
+    case class S3KeyClass(key: String) extends S3Key
+    def build(key: String, s3_key_prefix: Option[String]): S3Key = S3KeyClass(prefix(s3_key_prefix) + key)
+  }
 
   case class S3KeyGlob[T](globs: Map[String, T]) {
     def globMatch(s3Key: S3Key): Option[T] = {