Add requester pays configuration

thunder-project · Sep 11, 2014 · 1d83e71 · 1d83e71
1 parent 70cd242
commit 1d83e71
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/python/thunder/utils/ec2.py b/python/thunder/utils/ec2.py
@@ -63,6 +63,10 @@ def install_thunder(master, opts):
     access, secret = get_s3_keys()
     filled = configstring.replace('ACCESS', access).replace('SECRET', secret)
     ssh(master, opts, "sed -i'f' 's,.*</configuration>.*,"+filled+"&,' /root/ephemeral-hdfs/conf/core-site.xml")
+    # configure requester pays
+    ssh(master, opts, "touch /root/spark/conf/jets3t.properties")
+    ssh(master, opts, "echo 'httpclient.requester-pays-buckets-enabled = true' >> /root/spark/conf/jets3t.properties")
+    ssh(master, opts, "~/spark-ec2/copy-dir /root/spark/conf")
 
     print "\n\n"
     print "-------------------------------"