From 4dbcc0e19218d0a0df092635bbf6aea93e9fd8a0 Mon Sep 17 00:00:00 2001 From: Lauri Lehmijoki Date: Sat, 27 Jun 2015 16:09:58 +0300 Subject: [PATCH] Add s3_key_prefix --- README.md | 19 ++ changelog.md | 4 + src/main/scala/s3/website/CloudFront.scala | 2 +- src/main/scala/s3/website/S3.scala | 15 +- src/main/scala/s3/website/UploadHelper.scala | 8 +- src/main/scala/s3/website/model/Config.scala | 5 +- src/main/scala/s3/website/model/Site.scala | 9 +- src/main/scala/s3/website/model/push.scala | 11 +- src/main/scala/s3/website/package.scala | 12 +- src/test/scala/s3/website/S3WebsiteSpec.scala | 204 ++++++++++++++++++ 10 files changed, 272 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index fd3172b0..8fe2d2af 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,14 @@ redirects: music-files/promo.mp4: http://www.youtube.com/watch?v=dQw4w9WgXcQ ``` +On terminology: the left value is the redirect source and the right value is the redirect +target. For example above, *about.php* is the redirect source and */about.html* the target. + +If the `s3_key_prefix` setting is defined, it will be applied to the redirect +target if and only if the redirect target points to a site-local resource and +does not start with a slash. E.g., `about.php: about.html` will be translated +into `about.php: VALUE-OF-S3_KEY_PREFIX/about.html`. + #### Routing Rules You can configure more complex redirect rules by adding the following @@ -396,6 +404,17 @@ operation would actually do if run without the dry switch. You can use the dry run mode if you are unsure what kind of effects the `push` operation would cause to your live website. +### S3 website in a subdirectory of the bucket + +If your S3 website shares the same S3 bucket with other applications, you can +push your website into a "subdirectory" on the bucket. + +Define the subdirectory like so: + +```yaml +s3_key_prefix: your-subdirectory +``` + ## Migrating from v1 to v2 Please read the [release note](/changelog.md#200) on version 2. It contains diff --git a/changelog.md b/changelog.md index f35747f3..2beb62b2 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,10 @@ This project uses [Semantic Versioning](http://semver.org). +## NEXT + +* Add the `s3_key_prefix` setting + ## 2.10.0 * Support glob hashes in `cache_control` diff --git a/src/main/scala/s3/website/CloudFront.scala b/src/main/scala/s3/website/CloudFront.scala index a894765a..0a4cae53 100644 --- a/src/main/scala/s3/website/CloudFront.scala +++ b/src/main/scala/s3/website/CloudFront.scala @@ -81,7 +81,7 @@ object CloudFront { if (containsPotentialDefaultRootObject) Some("/") else None } val indexPath = config.cloudfront_invalidate_root collect { - case true if pushSuccessReports.nonEmpty => "/index.html" + case true if pushSuccessReports.nonEmpty => config.s3_key_prefix.map(prefix => s"/$prefix").getOrElse("") + "/index.html" } val invalidationPaths: Seq[String] = { diff --git a/src/main/scala/s3/website/S3.scala b/src/main/scala/s3/website/S3.scala index ef7eaef4..fb9cdf9f 100644 --- a/src/main/scala/s3/website/S3.scala +++ b/src/main/scala/s3/website/S3.scala @@ -116,21 +116,28 @@ object S3 { def awsS3Client(config: Config) = new AmazonS3Client(awsCredentials(config)) def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil, attempt: Attempt = 1) - (implicit config: Config, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushOptions: PushOptions): + (implicit site: Site, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushOptions: PushOptions): Future[Either[ErrorReport, Seq[S3File]]] = Future { logger.debug(nextMarker.fold ("Querying S3 files") {m => s"Querying more S3 files (starting from $m)"} ) - val objects: ObjectListing = s3Settings.s3Client(config).listObjects({ + val objects: ObjectListing = s3Settings.s3Client(site.config).listObjects({ val req = new ListObjectsRequest() - req.setBucketName(config.s3_bucket) + req.setBucketName(site.config.s3_bucket) nextMarker.foreach(req.setMarker) req }) objects } flatMap { (objects: ObjectListing) => - val s3Files = alreadyResolved ++ (objects.getObjectSummaries.toIndexedSeq.toSeq map (S3File(_))) + + /** + * We could filter the keys by prefix already on S3, but unfortunately s3_website test infrastructure does not currently support testing of that. + * Hence fetch all the keys from S3 and then filter by s3_key_prefix. + */ + def matchesPrefix(os: S3ObjectSummary) = site.config.s3_key_prefix.fold(true)(prefix => os.getKey.startsWith(prefix)) + + val s3Files = alreadyResolved ++ (objects.getObjectSummaries.filter(matchesPrefix).toIndexedSeq.toSeq map (S3File(_))) Option(objects.getNextMarker) .fold(Future(Right(s3Files)): Future[Either[ErrorReport, Seq[S3File]]]) // We've received all the S3 keys from the bucket { nextMarker => // There are more S3 keys on the bucket. Fetch them. diff --git a/src/main/scala/s3/website/UploadHelper.scala b/src/main/scala/s3/website/UploadHelper.scala index 046e2ec3..34092ea3 100644 --- a/src/main/scala/s3/website/UploadHelper.scala +++ b/src/main/scala/s3/website/UploadHelper.scala @@ -1,5 +1,6 @@ package s3.website +import s3.website.S3Key.isIgnoredBecauseOfPrefix import s3.website.model.Files.listSiteFiles import s3.website.model._ import s3.website.Ruby.rubyRegexMatches @@ -47,7 +48,7 @@ object UploadHelper { def resolveDeletes(s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect]) (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] = if (site.config.ignore_on_server exists ( - ignoreRegexes => ignoreRegexes.s3KeyRegexes exists( regex => regex matches S3Key(DELETE_NOTHING_MAGIC_WORD)) + ignoreRegexes => ignoreRegexes.s3KeyRegexes exists( regex => regex matches S3Key.build(DELETE_NOTHING_MAGIC_WORD, site.config.s3_key_prefix)) )) { logger.debug(s"Ignoring all files on the bucket, since the setting $DELETE_NOTHING_MAGIC_WORD is on.") Future(Right(Nil)) @@ -58,7 +59,10 @@ object UploadHelper { for { remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right } yield { - val keysToRetain = (localS3Keys ++ (redirects map { _.s3Key })).toSet + val keysIgnoredBecauseOf_s3_key_prefix = remoteS3Keys.filterNot(isIgnoredBecauseOfPrefix) + val keysToRetain = ( + localS3Keys ++ (redirects map { _.s3Key }) ++ keysIgnoredBecauseOf_s3_key_prefix + ).toSet remoteS3Keys filterNot { s3Key => val ignoreOnServer = site.config.ignore_on_server.exists(_ matches s3Key) if (ignoreOnServer) logger.debug(s"Ignoring $s3Key on server") diff --git a/src/main/scala/s3/website/model/Config.scala b/src/main/scala/s3/website/model/Config.scala index ef8497cc..bdc88410 100644 --- a/src/main/scala/s3/website/model/Config.scala +++ b/src/main/scala/s3/website/model/Config.scala @@ -20,6 +20,7 @@ case class Config( cache_control: Option[Either[String, S3KeyGlob[String]]], gzip: Option[Either[Boolean, Seq[String]]], gzip_zopfli: Option[Boolean], + s3_key_prefix: Option[String], ignore_on_server: Option[S3KeyRegexes], exclude_from_upload: Option[S3KeyRegexes], s3_reduced_redundancy: Option[Boolean], @@ -119,14 +120,14 @@ object Config { } } - def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[S3Key, String]]] = { + def loadRedirects(s3_key_prefix: Option[String])(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[S3Key, String]]] = { val key = "redirects" val yamlValue = for { redirectsOption <- loadOptionalValue(key) redirectsOption <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap)) } yield Right(redirectsOption.map( redirects => redirects.map( - ((key: String, value: String) => (S3Key(key), value)).tupled + ((key: String, value: String) => (S3Key.build(key, s3_key_prefix), value)).tupled ) )) diff --git a/src/main/scala/s3/website/model/Site.scala b/src/main/scala/s3/website/model/Site.scala index 8d371c38..ad6cba50 100644 --- a/src/main/scala/s3/website/model/Site.scala +++ b/src/main/scala/s3/website/model/Site.scala @@ -15,7 +15,10 @@ import s3.website.model.Config.UnsafeYaml import scala.util.Success case class Site(rootDirectory: File, config: Config) { - def resolveS3Key(file: File) = S3Key(file.getAbsolutePath.replace(rootDirectory.getAbsolutePath, "").replace(File.separator,"/").replaceFirst("^/", "")) + def resolveS3Key(file: File) = S3Key.build( + file.getAbsolutePath.replace(rootDirectory.getAbsolutePath, "").replace(File.separator,"/").replaceFirst("^/", ""), + config.s3_key_prefix + ) } object Site { @@ -40,13 +43,14 @@ object Site { gzip <- loadOptionalBooleanOrStringSeq("gzip").right gzip_zopfli <- loadOptionalBoolean("gzip_zopfli").right extensionless_mime_type <- loadOptionalString("extensionless_mime_type").right + s3_key_prefix <- loadOptionalString("s3_key_prefix").right ignore_on_server <- loadOptionalS3KeyRegexes("ignore_on_server").right exclude_from_upload <- loadOptionalS3KeyRegexes("exclude_from_upload").right s3_reduced_redundancy <- loadOptionalBoolean("s3_reduced_redundancy").right cloudfront_distribution_id <- loadOptionalString("cloudfront_distribution_id").right cloudfront_invalidate_root <- loadOptionalBoolean("cloudfront_invalidate_root").right concurrency_level <- loadOptionalInt("concurrency_level").right - redirects <- loadRedirects.right + redirects <- loadRedirects(s3_key_prefix).right treat_zero_length_objects_as_redirects <- loadOptionalBoolean("treat_zero_length_objects_as_redirects").right } yield { gzip_zopfli.foreach(_ => logger.info( @@ -66,6 +70,7 @@ object Site { cache_control, gzip, gzip_zopfli, + s3_key_prefix, ignore_on_server = ignore_on_server, exclude_from_upload = exclude_from_upload, s3_reduced_redundancy, diff --git a/src/main/scala/s3/website/model/push.scala b/src/main/scala/s3/website/model/push.scala index c8b1db8a..0d967533 100644 --- a/src/main/scala/s3/website/model/push.scala +++ b/src/main/scala/s3/website/model/push.scala @@ -132,7 +132,7 @@ object Files { val excludeByConfig = site.config.exclude_from_upload exists { _.s3KeyRegexes.exists(_ matches s3Key) } - val neverUpload = "s3_website.yml" :: ".env" :: Nil map (k => S3Key(k)) + val neverUpload = "s3_website.yml" :: ".env" :: Nil map (k => S3Key.build(k, site.config.s3_key_prefix)) val doNotUpload = excludeByConfig || (neverUpload contains s3Key) if (doNotUpload) logger.debug(s"Excluded $s3Key from upload") doNotUpload @@ -157,7 +157,7 @@ object Redirect { val redirectSettings = config.redirects.fold(Nil: Seq[RedirectSetting]) { sourcesToTargets => sourcesToTargets.foldLeft(Seq(): Seq[RedirectSetting]) { (redirects, sourceToTarget) => - redirects :+ RedirectSetting(sourceToTarget._1, applySlashIfNeeded(sourceToTarget._2)) + redirects :+ RedirectSetting(sourceToTarget._1, applyRedirectRules(sourceToTarget._2)) } } def redirectsWithExistsOnS3Info = @@ -178,13 +178,13 @@ object Redirect { allConfiguredRedirects } - private def applySlashIfNeeded(redirectTarget: String) = { + private def applyRedirectRules(redirectTarget: String)(implicit config: Config) = { val isExternalRedirect = redirectTarget.matches("https?:\\/\\/.*") val isInSiteRedirect = redirectTarget.startsWith("/") if (isInSiteRedirect || isExternalRedirect) redirectTarget else - "/" + redirectTarget // let the user have redirect settings like "index.php: index.html" in s3_website.ml + s"${config.s3_key_prefix.map(prefix => s"/$prefix").getOrElse("")}/$redirectTarget" } def apply(redirectSetting: RedirectSetting, needsUpload: Boolean): Redirect = @@ -194,5 +194,6 @@ object Redirect { case class S3File(s3Key: S3Key, md5: MD5, size: Long) object S3File { - def apply(summary: S3ObjectSummary): S3File = S3File(S3Key(summary.getKey), summary.getETag, summary.getSize) + def apply(summary: S3ObjectSummary)(implicit site: Site): S3File = + S3File(S3Key.build(summary.getKey, None), summary.getETag, summary.getSize) } diff --git a/src/main/scala/s3/website/package.scala b/src/main/scala/s3/website/package.scala index b7439c4c..04d70f0d 100644 --- a/src/main/scala/s3/website/package.scala +++ b/src/main/scala/s3/website/package.scala @@ -60,9 +60,19 @@ package object website { def matches(s3Key: S3Key) = rubyRegexMatches(s3Key.key, keyRegex.pattern.pattern()) } - case class S3Key(key: String) { + trait S3Key { + val key: String override def toString = key } + + object S3Key { + def prefix(s3_key_prefix: Option[String]) = s3_key_prefix.map(prefix => if (prefix.endsWith("/")) prefix else prefix + "/").getOrElse("") + + def isIgnoredBecauseOfPrefix(s3Key: S3Key)(implicit site: Site) = s3Key.key.startsWith(prefix(site.config.s3_key_prefix)) + + case class S3KeyClass(key: String) extends S3Key + def build(key: String, s3_key_prefix: Option[String]): S3Key = S3KeyClass(prefix(s3_key_prefix) + key) + } case class S3KeyGlob[T](globs: Map[String, T]) { def globMatch(s3Key: S3Key): Option[T] = { diff --git a/src/test/scala/s3/website/S3WebsiteSpec.scala b/src/test/scala/s3/website/S3WebsiteSpec.scala index 4adb79fd..43ca2d83 100644 --- a/src/test/scala/s3/website/S3WebsiteSpec.scala +++ b/src/test/scala/s3/website/S3WebsiteSpec.scala @@ -83,6 +83,14 @@ class S3WebsiteSpec extends Specification { noUploadsOccurred must beTrue } + "not upload a file if it has not changed and s3_key_prefix is defined" in new BasicSetup { + config = "s3_key_prefix: test" + setLocalFileWithContent(("index.html", "
hello
")) + setS3File("test/index.html", md5Hex("
hello
")) + push() + noUploadsOccurred must beTrue + } + "detect a changed file even though another file has the same contents as the changed file" in new BasicSetup { setLocalFilesWithContent(("1.txt", "foo"), ("2.txt", "foo")) setS3File("1.txt", md5Hex("bar")) @@ -110,6 +118,27 @@ class S3WebsiteSpec extends Specification { sentDelete must equalTo("old.html") } + "delete files that match the s3_key_prefix" in new BasicSetup { + config = "s3_key_prefix: production" + setS3File("production/old.html", md5Hex("

old text

")) + push() + sentDelete must equalTo("production/old.html") + } + + "retain files that do not match the s3_key_prefix" in new BasicSetup { + config = "s3_key_prefix: production" + setS3File("old.html", md5Hex("

old text

")) + push() + noDeletesOccurred + } + + "retain files that do not match the s3_key_prefix" in new BasicSetup { + config = "s3_key_prefix: test" + setS3File("test1.html") + push() + noDeletesOccurred + } + "try again if the upload fails" in new BasicSetup { setLocalFile("index.html") uploadFailsAndThenSucceeds(howManyFailures = 5) @@ -230,6 +259,17 @@ class S3WebsiteSpec extends Specification { push() sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/" :: "/maybe-index.html" :: Nil).sorted) } + + "work with s3_key_prefix" in new BasicSetup { + config = """ + |cloudfront_distribution_id: EGM1J2JJX9Z + |s3_key_prefix: production + """.stripMargin + setLocalFile("index.html") + setOutdatedS3Keys("production/index.html") + push() + sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/production/index.html" :: Nil).sorted) + } } "cloudfront_invalidate_root: true" should { @@ -255,6 +295,20 @@ class S3WebsiteSpec extends Specification { sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq must contain("/index.html") } + "treat the s3_key_prefix as the root path" in new BasicSetup { + config = """ + |cloudfront_distribution_id: EGM1J2JJX9Z + |cloudfront_invalidate_root: true + |s3_key_prefix: test + """.stripMargin + setLocalFile("articles/index.html") + setOutdatedS3Keys("test/articles/index.html") + push() + sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo( + ("/test/index.html" :: "/test/articles/" :: Nil).sorted + ) + } + "not invalidate anything if there was nothing to push" in new BasicSetup { config = """ |cloudfront_distribution_id: EGM1J2JJX9Z @@ -333,12 +387,35 @@ class S3WebsiteSpec extends Specification { } } + "s3_key_prefix in config" should { + "apply the prefix into all the S3 keys" in new BasicSetup { + config = "s3_key_prefix: production" + setLocalFile("index.html") + push() + sentPutObjectRequest.getKey must equalTo("production/index.html") + } + + "work with slash" in new BasicSetup { + config = "s3_key_prefix: production/" + setLocalFile("index.html") + push() + sentPutObjectRequest.getKey must equalTo("production/index.html") + } + } + "s3_website.yml file" should { "never be uploaded" in new BasicSetup { setLocalFile("s3_website.yml") push() noUploadsOccurred must beTrue } + + "never be uploaded even when s3_key_prefix is defined" in new BasicSetup { + config = "s3_key_prefix: production" + setLocalFile("s3_website.yml") + push() + noUploadsOccurred must beTrue + } } ".env file" should { // The .env file is the https://github.com/bkeepers/dotenv file @@ -347,6 +424,13 @@ class S3WebsiteSpec extends Specification { push() noUploadsOccurred must beTrue } + + "never be uploaded even when s3_key_prefix is defined" in new BasicSetup { + config = "s3_key_prefix: production" + setLocalFile(".env") + push() + noUploadsOccurred must beTrue + } } "exclude_from_upload: string" should { @@ -356,6 +440,16 @@ class S3WebsiteSpec extends Specification { push() noUploadsOccurred must beTrue } + + "work with s3_key_prefix" in new BasicSetup { + config = """ + |s3_key_prefix: production + |exclude_from_upload: hello.txt + """.stripMargin + setLocalFile("hello.txt") + push() + noUploadsOccurred must beTrue + } } """ @@ -373,6 +467,17 @@ class S3WebsiteSpec extends Specification { push() noUploadsOccurred must beTrue } + + "work with s3_key_prefix" in new BasicSetup { + config = """ + |s3_key_prefix: production + |exclude_from_upload: + |- hello.txt + """.stripMargin + setLocalFile("hello.txt") + push() + noUploadsOccurred must beTrue + } } "ignore_on_server: value" should { @@ -389,6 +494,16 @@ class S3WebsiteSpec extends Specification { push() noDeletesOccurred must beTrue } + + "work with s3_key_prefix" in new BasicSetup { + config = """ + |s3_key_prefix: production + |ignore_on_server: hello.txt + """.stripMargin + setS3File("hello.txt") + push() + noDeletesOccurred must beTrue + } } "ignore_on_server: _DELETE_NOTHING_ON_THE_S3_BUCKET_" should { @@ -400,6 +515,16 @@ class S3WebsiteSpec extends Specification { push() noDeletesOccurred } + + "work with s3_key_prefix" in new BasicSetup { + config = s""" + |s3_key_prefix: production + |ignore_on_server: $DELETE_NOTHING_MAGIC_WORD + """.stripMargin + setS3File("file.txt") + push() + noDeletesOccurred + } } """ @@ -426,6 +551,17 @@ class S3WebsiteSpec extends Specification { push() noDeletesOccurred must beTrue } + + "work with s3_key_prefix" in new BasicSetup { + config = """ + |s3_key_prefix: production + |ignore_on_server: + |- hello.* + """.stripMargin + setS3File("hello.txt") + push() + noDeletesOccurred must beTrue + } } "error message" should { @@ -485,6 +621,17 @@ class S3WebsiteSpec extends Specification { sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, no-transform, max-age=1200, s-maxage=1200") } + "work with s3_key_prefix" in new BasicSetup { + config = + """ + |cache_control: public, no-transform, max-age=1200, s-maxage=1200 + |s3_key_prefix: foo + """.stripMargin + setLocalFile("index.html") + push() + sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, no-transform, max-age=1200, s-maxage=1200") + } + "should take precedence over max_age" in new BasicSetup { config = """ |max_age: 120 @@ -568,6 +715,18 @@ class S3WebsiteSpec extends Specification { push() sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, must-revalidate, max-age=120") } + + "work with s3_key_prefix" in new BasicSetup { + config = + """ + |cache_control: + | "*.html": public, no-transform, max-age=1200, s-maxage=1200 + |s3_key_prefix: foo + """.stripMargin + setLocalFile("index.html") + push() + sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, no-transform, max-age=1200, s-maxage=1200") + } } "cache control" can { @@ -586,6 +745,17 @@ class S3WebsiteSpec extends Specification { sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60") } + "work with s3_key_prefix" in new BasicSetup { + config = + """ + |max_age: 60 + |s3_key_prefix: test + """.stripMargin + setLocalFile("index.html") + push() + sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60") + } + "supports all valid URI characters in the glob setting" in new BasicSetup { config = """ |max_age: @@ -665,6 +835,18 @@ class S3WebsiteSpec extends Specification { sentPutObjectRequests.find(_.getKey == "assets/jquery.js").get.getMetadata.getCacheControl must equalTo("max-age=150") sentPutObjectRequests.find(_.getKey == "assets/picture.gif").get.getMetadata.getCacheControl must equalTo("max-age=86400") } + + "work with s3_key_prefix" in new BasicSetup { + config = + """ + |max_age: + | "*.html": 60 + |s3_key_prefix: test + """.stripMargin + setLocalFile("index.html") + push() + sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60") + } } "s3_reduced_redundancy: true in config" should { @@ -695,6 +877,28 @@ class S3WebsiteSpec extends Specification { sentPutObjectRequest.getRedirectLocation must equalTo("/index.html") } + "refer to site root when the s3_key_prefix is defined and the redirect target starts with a slash" in new BasicSetup { + config = """ + |s3_key_prefix: production + |redirects: + | index.php: /index.html + """.stripMargin + push() + sentPutObjectRequest.getKey must equalTo("production/index.php") + sentPutObjectRequest.getRedirectLocation must equalTo("/index.html") + } + + "use s3_key_prefix as the root when the redirect target does not start with a slash" in new BasicSetup { + config = """ + |s3_key_prefix: production + |redirects: + | index.php: index.html + """.stripMargin + push() + sentPutObjectRequest.getKey must equalTo("production/index.php") + sentPutObjectRequest.getRedirectLocation must equalTo("/production/index.html") + } + "add slash to the redirect target" in new BasicSetup { config = """ |redirects: