{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# S3" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2017-02-06T00:40:11.039426", "start_time": "2017-02-06T00:40:10.851600" }, "collapsed": false }, "outputs": [ { "ename": "ImportError", "evalue": "No module named 'boto3'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mboto3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mbotocore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexceptions\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mClientError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mclass\u001b[0m \u001b[0mS3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mImportError\u001b[0m: No module named 'boto3'" ] } ], "source": [ "import boto3\n", "from botocore.exceptions import ClientError\n", "\n", "\n", "class S3():\n", " def __init__(self, setting):\n", " # proxy設定が必要な場合は事前に設定\n", " session = boto3.session.Session(aws_access_key_id=setting.accesskey,\n", " aws_secret_access_key=setting.secretkey,\n", " region_name=setting.region)\n", " self.session = session.session\n", " self.s3_client = self.session.client('s3')\n", " self.s3 = self.session.resource('s3')\n", " self.bucket = self.s3.Bucket(self.setting.bucket_name)\n", "\n", " def get_available_resources(self):\n", " # ['cloudformation', 'dynamodb', 'ec2', 'glacier', 'iam', 'opsworks', 's3', 'sns', 'sqs']\n", " self.session.get_available_resources()\n", "\n", " def get_available_services(self):\n", " \"\"\"一部リソース\n", " ['autoscaling', 'cloudfront', 'cloudhsm', 'cloudsearch',\n", " 'cloudsearchdomain', 'cloudtrail',\n", " 'cloudwatch', 'codecommit', 'codedeploy', 'codepipeline',\n", " 'cognito-identity', 'cognito-sync',\n", " 'config', 'datapipeline', 'devicefarm', 'directconnect', 'ds',\n", " 'dynamodbstreams', 'ecs', 'efs',\n", " 'elasticache', 'elasticbeanstalk', 'elastictranscoder',\n", " 'elb', 'emr', 'importexport', 'kinesis',\n", " 'kms', 'lambda', 'logs', 'machinelearning',\n", " 'rds', 'redshift', 'route53', 'route53domains', 'sdb',\n", " 'ses', 'ssm', 'storagegateway', 'sts', 'support', 'swf', 'workspaces']\n", " \"\"\"\n", " self.session.get_available_services()\n", "\n", " def upload(self, file_path, key, bucket_name=None):\n", " if bucket_name:\n", " self.s3_client.upload_file(file_path, bucket_name, key)\n", " else:\n", " with open(file_path, 'rb') as f:\n", " r = self.bucket.put_object(Key=key, Body=f)\n", " print(r)\n", " # s3.Object(bucket_name='', key='')\n", "\n", " def download(self, key, download_path, bucket_name=None):\n", " if not bucket_name:\n", " bucket_name = self.setting.bucket_name\n", " try:\n", " self.s3_client.download_file(bucket_name, key, download_path)\n", " except ClientError as e:\n", " print(e)\n", " # print(e.parsed_response, e.operation_name)\n", " raise e\n", "\n", " def exists_bucket(self):\n", " exists = True\n", " try:\n", " self.s3.meta.client.head_bucket(Bucket=self.bucket_name)\n", " except Exception as e:\n", " # If a client error is thrown, then check that it was a 404 error.\n", " # If it was a 404 error, then the bucket does not exist.\n", " error_code = int(e.response['Error']['Code'])\n", " # print(e.response)\n", " if error_code == 404:\n", " exists = False\n", " return exists\n", "\n", " def object_all(self):\n", " for v in self.bucket.objects.all():\n", " print(v)\n", "\n", " def bucket_dir(self, key):\n", " bucket_dir = self.s3.Bucket(self.bucket_name + key)\n", " print(dir(bucket_dir.objects))\n", " # s3.Bucket.objectsCollectionManager(\n", " # s3.Bucket(name='/'), s3.ObjectSummary)\n", " # ['all', 'delete', 'filter', 'iterator', 'limit', 'page_size', 'pages']\n", " for v in bucket_dir.objects.pages():\n", " print(v)\n", "\n", " def list_objects(self, prefix):\n", " response = self.s3_c.list_objects(Bucket=self.bucket_name, Prefix=prefix)\n", " if 'Contents' in response:\n", " # 該当する key がないと response に 'Contents' が含まれない\n", " for content in response['Contents']:\n", " # \n", " print(content)\n", " print(type(content))\n", " print(dir(content))\n", "\n", " def s3_object(self, key):\n", " try:\n", " obj = self.s3.Object(bucket_name=self.bucket_name, key=key)\n", " # key not exists, directory, file\n", " print(dir(obj))\n", " # \n", " # ['Acl', 'Bucket', 'MultipartUpload', 'Version']\n", " # ['accept_ranges', 'bucket_name', 'cache_control',\n", " # 'content_disposition', 'content_encoding', 'content_language',\n", " # content_length', 'content_type', 'copy_from',\n", " # 'delete', 'delete_marker', 'e_tag', 'expiration', 'expires', 'get',\n", " # initiate_multipart_upload', 'key', 'last_modified',\n", " # 'load', 'meta', 'metadata', 'missing_meta', 'put', 'reload',\n", " # replication_status', 'request_charged', 'restore',\n", " # 'server_side_encryption', 'sse_customer_algorithm', 'sse_customer_key_md5',\n", " # ssekms_key_id', 'storage_class', 'version_id',\n", " # 'wait_until_exists', 'wait_until_not_exists', 'website_redirect_location']\n", "\n", " # content not found when obj is dir\n", " # obj.content_length\n", " # obj.content_type\n", " # obj.content_encoding\n", " # obj.content_language\n", " # obj.content_disposition\n", "\n", " print(obj.key)\n", " print(obj.meta)\n", " print(obj.metadata)\n", "\n", " # response\n", " response = obj.get()\n", " # {'ContentType': 'text/tab-separated-values',\n", " # 'LastModified': datetime.datetime(2015, 8, 10, 4, 35, 22, tzinfo=tzutc()),\n", " # 'ETag': '\"ff278edf2bd312bee37f863f54bd29e6\"',\n", " # 'ResponseMetadata': {\n", " # 'HostId':\n", " # '',\n", " # 'RequestId': '', 'HTTPStatusCode': 200},\n", " # 'Body': ,\n", " # 'ContentLength': 1103, 'Metadata': {}, 'AcceptRanges': 'bytes'}\n", " # \n", "\n", " data = response['Body'].read()\n", " print(data)\n", " except Exception as e:\n", " # except ClientError as e: -> \n", " print(e)\n", "\n", " def buckets_all(self):\n", " for bucket in self.s3.buckets.all():\n", " pass\n" ] } ], "metadata": { "hide_input": false, "kernelspec": { "display_name": "Python [default]", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.5.2" }, "toc": { "toc_cell": false, "toc_number_sections": true, "toc_threshold": 6, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 2 }