C++과 Amazon S3를 연동해 대규모 파일 업로드 및 다운로드 구현하기

sudo apt update
sudo apt install -y cmake git g++ clang libssl-dev

git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.git
cd aws-sdk-cpp

mkdir build
cd build
cmake .. -DBUILD_ONLY="s3"
cmake --build . --target install

aws configure

export AWS_ACCESS_KEY_ID="your-access-key"
export AWS_SECRET_ACCESS_KEY="your-secret-key"
export AWS_REGION="us-east-1"

set AWS_ACCESS_KEY_ID=your-access-key
set AWS_SECRET_ACCESS_KEY=your-secret-key
set AWS_REGION=us-east-1

cmake_minimum_required(VERSION 3.14)
project(S3Example)

find_package(AWSSDK REQUIRED COMPONENTS s3)

add_executable(s3_example main.cpp)
target_link_libraries(s3_example PRIVATE ${AWSSDK_LIBRARIES})

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/PutObjectRequest.h>

#include <aws/core/Aws.h>
#include <iostream>

int main() {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    std::cout << "AWS SDK for C++ Initialized Successfully!" << std::endl;
    Aws::ShutdownAPI(options);
    return 0;
}

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateBucketRequest.h>
#include <iostream>

int main() {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    {
        Aws::S3::S3Client s3_client;
        Aws::S3::Model::CreateBucketRequest request;
        request.SetBucket("my-cpp-s3-bucket");

        auto outcome = s3_client.CreateBucket(request);
        if (outcome.IsSuccess()) {
            std::cout << "S3 Bucket created successfully!" << std::endl;
        } else {
            std::cerr << "Error creating S3 Bucket: " 
                      << outcome.GetError().GetMessage() << std::endl;
        }
    }
    Aws::ShutdownAPI(options);
    return 0;
}

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket",
                "s3:GetObject",
                "s3:PutObject"
            ],
            "Resource": [
                "arn:aws:s3:::my-cpp-s3-bucket",
                "arn:aws:s3:::my-cpp-s3-bucket/*"
            ]
        }
    ]
}

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": "*",
            "Action": "s3:GetObject",
            "Resource": "arn:aws:s3:::my-cpp-s3-bucket/*",
            "Condition": {
                "IpAddress": {
                    "aws:SourceIp": "203.0.113.0/24"
                }
            }
        }
    ]
}

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <iostream>

int main() {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    {
        Aws::S3::S3Client s3_client;
        Aws::S3::Model::ListObjectsRequest request;
        request.SetBucket("my-cpp-s3-bucket");

        auto outcome = s3_client.ListObjects(request);
        if (outcome.IsSuccess()) {
            std::cout << "S3 Bucket Contents:" << std::endl;
            for (const auto& obj : outcome.GetResult().GetContents()) {
                std::cout << obj.GetKey() << std::endl;
            }
        } else {
            std::cerr << "Error listing S3 Bucket contents: " 
                      << outcome.GetError().GetMessage() << std::endl;
        }
    }
    Aws::ShutdownAPI(options);
    return 0;
}

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <fstream>
#include <iostream>

void UploadFileToS3(const std::string& bucketName, const std::string& filePath, const std::string& objectKey) {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    {
        Aws::S3::S3Client s3_client;

        // 파일을 읽어오기 위한 입력 스트림
        std::shared_ptr<Aws::IOStream> inputData = 
            Aws::MakeShared<Aws::FStream>("SampleAllocationTag",
                                          filePath.c_str(),
                                          std::ios_base::in | std::ios_base::binary);

        // S3 업로드 요청 생성
        Aws::S3::Model::PutObjectRequest request;
        request.SetBucket(bucketName);
        request.SetKey(objectKey);
        request.SetBody(inputData);

        // 업로드 실행
        auto outcome = s3_client.PutObject(request);
        if (outcome.IsSuccess()) {
            std::cout << "File uploaded successfully: " << objectKey << std::endl;
        } else {
            std::cerr << "Upload failed: " << outcome.GetError().GetMessage() << std::endl;
        }
    }
    Aws::ShutdownAPI(options);
}

int main() {
    UploadFileToS3("my-cpp-s3-bucket", "test.txt", "uploaded_test.txt");
    return 0;
}

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/CreateMultipartUploadRequest.h>
#include <aws/s3/model/UploadPartRequest.h>
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
#include <fstream>
#include <iostream>
#include <vector>

const int PART_SIZE = 5 * 1024 * 1024; // 5MB 청크 크기

void MultipartUploadToS3(const std::string& bucketName, const std::string& filePath, const std::string& objectKey) {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    {
        Aws::S3::S3Client s3_client;

        // 멀티파트 업로드 생성
        Aws::S3::Model::CreateMultipartUploadRequest createRequest;
        createRequest.SetBucket(bucketName);
        createRequest.SetKey(objectKey);
        auto createOutcome = s3_client.CreateMultipartUpload(createRequest);

        if (!createOutcome.IsSuccess()) {
            std::cerr << "Multipart Upload Init Failed: " << createOutcome.GetError().GetMessage() << std::endl;
            return;
        }

        std::string uploadId = createOutcome.GetResult().GetUploadId();
        std::ifstream file(filePath, std::ios::binary);
        std::vector<Aws::S3::Model::CompletedPart> completedParts;
        int partNumber = 1;
        char buffer[PART_SIZE];

        while (file.read(buffer, PART_SIZE) || file.gcount() > 0) {
            Aws::S3::Model::UploadPartRequest uploadPartRequest;
            uploadPartRequest.SetBucket(bucketName);
            uploadPartRequest.SetKey(objectKey);
            uploadPartRequest.SetPartNumber(partNumber);
            uploadPartRequest.SetUploadId(uploadId);

            auto stream = Aws::MakeShared<Aws::StringStream>("SampleTag");
            stream->write(buffer, file.gcount());
            uploadPartRequest.SetBody(stream);

            auto uploadPartOutcome = s3_client.UploadPart(uploadPartRequest);
            if (uploadPartOutcome.IsSuccess()) {
                Aws::S3::Model::CompletedPart completedPart;
                completedPart.SetPartNumber(partNumber);
                completedPart.SetETag(uploadPartOutcome.GetResult().GetETag());
                completedParts.push_back(completedPart);
            } else {
                std::cerr << "Part Upload Failed: " << uploadPartOutcome.GetError().GetMessage() << std::endl;
                return;
            }

            partNumber++;
        }

        // 업로드 완료 요청
        Aws::S3::Model::CompleteMultipartUploadRequest completeRequest;
        completeRequest.SetBucket(bucketName);
        completeRequest.SetKey(objectKey);
        completeRequest.SetUploadId(uploadId);

        Aws::S3::Model::CompletedMultipartUpload completedUpload;
        completedUpload.SetParts(completedParts);
        completeRequest.SetMultipartUpload(completedUpload);

        auto completeOutcome = s3_client.CompleteMultipartUpload(completeRequest);
        if (completeOutcome.IsSuccess()) {
            std::cout << "Multipart Upload Completed: " << objectKey << std::endl;
        } else {
            std::cerr << "Multipart Upload Failed: " << completeOutcome.GetError().GetMessage() << std::endl;
        }
    }
    Aws::ShutdownAPI(options);
}

int main() {
    MultipartUploadToS3("my-cpp-s3-bucket", "large_file.bin", "uploaded_large_file.bin");
    return 0;
}

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <fstream>
#include <iostream>

void DownloadFileFromS3(const std::string& bucketName, const std::string& objectKey, const std::string& savePath) {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    {
        Aws::S3::S3Client s3_client;

        // 다운로드 요청 생성
        Aws::S3::Model::GetObjectRequest request;
        request.SetBucket(bucketName);
        request.SetKey(objectKey);

        // S3에서 파일 다운로드
        auto outcome = s3_client.GetObject(request);
        if (outcome.IsSuccess()) {
            std::ofstream outputFile(savePath, std::ios::binary);
            outputFile << outcome.GetResult().GetBody().rdbuf();
            std::cout << "File downloaded successfully: " << savePath << std::endl;
        } else {
            std::cerr << "Download failed: " << outcome.GetError().GetMessage() << std::endl;
        }
    }
    Aws::ShutdownAPI(options);
}

int main() {
    DownloadFileFromS3("my-cpp-s3-bucket", "uploaded_test.txt", "downloaded_test.txt");
    return 0;
}

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <fstream>
#include <iostream>

void DownloadPartFromS3(const std::string& bucketName, const std::string& objectKey, const std::string& savePath, long startByte, long endByte) {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    {
        Aws::S3::S3Client s3_client;
        Aws::S3::Model::GetObjectRequest request;
        request.SetBucket(bucketName);
        request.SetKey(objectKey);

        // 범위를 설정하여 부분 다운로드 수행
        request.SetRange("bytes=" + std::to_string(startByte) + "-" + std::to_string(endByte));

        auto outcome = s3_client.GetObject(request);
        if (outcome.IsSuccess()) {
            std::ofstream outputFile(savePath, std::ios::binary | std::ios::app);
            outputFile << outcome.GetResult().GetBody().rdbuf();
            std::cout << "Downloaded bytes " << startByte << " to " << endByte << std::endl;
        } else {
            std::cerr << "Partial download failed: " << outcome.GetError().GetMessage() << std::endl;
        }
    }
    Aws::ShutdownAPI(options);
}

int main() {
    DownloadPartFromS3("my-cpp-s3-bucket", "large_file.bin", "partial_download.bin", 0, 1048575); // 1MB 다운로드
    return 0;
}

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <fstream>
#include <iostream>
#include <thread>
#include <vector>

const int PART_SIZE = 5 * 1024 * 1024; // 5MB 단위 다운로드

void DownloadChunk(const std::string& bucketName, const std::string& objectKey, long startByte, long endByte, int partNumber) {
    Aws::SDKOptions options;
    Aws::InitAPI(options);
    {
        Aws::S3::S3Client s3_client;
        Aws::S3::Model::GetObjectRequest request;
        request.SetBucket(bucketName);
        request.SetKey(objectKey);
        request.SetRange("bytes=" + std::to_string(startByte) + "-" + std::to_string(endByte));

        auto outcome = s3_client.GetObject(request);
        if (outcome.IsSuccess()) {
            std::ofstream outputFile("chunk_" + std::to_string(partNumber) + ".bin", std::ios::binary);
            outputFile << outcome.GetResult().GetBody().rdbuf();
            std::cout << "Chunk " << partNumber << " downloaded." << std::endl;
        } else {
            std::cerr << "Chunk " << partNumber << " download failed: " 
                      << outcome.GetError().GetMessage() << std::endl;
        }
    }
    Aws::ShutdownAPI(options);
}

void ParallelDownloadFromS3(const std::string& bucketName, const std::string& objectKey, long fileSize) {
    int numParts = fileSize / PART_SIZE + (fileSize % PART_SIZE != 0 ? 1 : 0);
    std::vector<std::thread> threads;

    for (int i = 0; i < numParts; ++i) {
        long startByte = i * PART_SIZE;
        long endByte = std::min(startByte + PART_SIZE - 1, fileSize - 1);
        threads.emplace_back(DownloadChunk, bucketName, objectKey, startByte, endByte, i);
    }

    for (auto& t : threads) {
        t.join();
    }

    std::cout << "All chunks downloaded. Merging files..." << std::endl;
}

int main() {
    ParallelDownloadFromS3("my-cpp-s3-bucket", "large_file.bin", 50 * 1024 * 1024); // 50MB 파일 다운로드
    return 0;
}

Aws::S3::Model::UploadPartRequest uploadPartRequest;
uploadPartRequest.SetPartNumber(partNumber);
uploadPartRequest.SetBody(dataStream);
auto uploadPartOutcome = s3_client.UploadPart(uploadPartRequest);

if (!uploadPartOutcome.IsSuccess()) {
    std::cerr << "Upload failed, retrying..." << std::endl;
    // 재시도 로직 추가 가능
}

std::vector<std::thread> threads;
for (int i = 0; i < numParts; ++i) {
    long startByte = i * PART_SIZE;
    long endByte = std::min(startByte + PART_SIZE - 1, fileSize - 1);
    threads.emplace_back(DownloadChunk, bucketName, objectKey, startByte, endByte, i);
}
for (auto& t : threads) {
    t.join();
}

Aws::Client::ClientConfiguration config;
config.endpointOverride = "my-bucket.s3-accelerate.amazonaws.com";
Aws::S3::S3Client s3_client(config);

Aws::Client::ClientConfiguration config;
config.maxConnections = 10; // 동시 요청 개수
Aws::S3::S3Client s3_client(config);

gzip -c large_file.bin > large_file.bin.gz
zstd -c large_file.bin > large_file.bin.zst

Aws::S3::Model::PutObjectRequest request;
request.SetContentEncoding("gzip");

request.SetContentType("application/x-parquet");

Aws::Client::ClientConfiguration config;
config.endpointOverride = "d123456.cloudfront.net"; // CloudFront URL
Aws::S3::S3Client s3_client(config);

Aws::S3::Model::HeadObjectRequest request;
request.SetBucket("my-bucket");
request.SetKey("large_file.bin");
auto outcome = s3_client.HeadObject(request);
if (outcome.IsSuccess()) {
    std::string eTag = outcome.GetResult().GetETag();
    // 기존 파일의 ETag와 비교하여 캐싱 여부 결정
}

Aws::Client::ClientConfiguration config;
config.requestTimeoutMs = 30000; // 30초
config.connectTimeoutMs = 10000; // 10초
config.retryStrategy = std::make_shared<Aws::Client::DefaultRetryStrategy>(3, 200);
Aws::S3::S3Client s3_client(config);

int maxRetries = 5;
int delay = 1000; // 1초

for (int attempt = 1; attempt <= maxRetries; attempt++) {
    auto outcome = s3_client.PutObject(request);
    if (outcome.IsSuccess()) {
        std::cout << "Upload successful!" << std::endl;
        break;
    } else {
        std::cerr << "Attempt " << attempt << " failed: " 
                  << outcome.GetError().GetMessage() << std::endl;
        std::this_thread::sleep_for(std::chrono::milliseconds(delay));
        delay *= 2; // 지수 증가
    }
}

Aws::S3::Model::GetObjectRequest request;
request.SetBucket("my-bucket");
request.SetKey("file.txt");

auto outcome = s3_client.GetObject(request);
if (!outcome.IsSuccess()) {
    std::cerr << "Error: " << outcome.GetError().GetMessage() << std::endl;
    if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) {
        std::cerr << "Check your AWS credentials and permissions!" << std::endl;
    }
}

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:PutObject",
                "s3:GetObject"
            ],
            "Resource": "arn:aws:s3:::my-bucket/*"
        }
    ]
}

Aws::S3::Model::HeadBucketRequest request;
request.SetBucket("my-bucket");

auto outcome = s3_client.HeadBucket(request);
if (!outcome.IsSuccess()) {
    std::cerr << "Bucket access denied: " << outcome.GetError().GetMessage() << std::endl;
}

if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::THROTTLING) {
    std::cerr << "Request throttled. Retrying..." << std::endl;
    std::this_thread::sleep_for(std::chrono::seconds(2)); // 2초 대기 후 재시도
}

auto eTag = putOutcome.GetResult().GetETag();
std::cout << "Upload complete. ETag: " << eTag << std::endl;

Aws::S3::Model::HeadObjectRequest headRequest;
headRequest.SetBucket("my-bucket");
headRequest.SetKey("file.txt");

auto headOutcome = s3_client.HeadObject(headRequest);
if (headOutcome.IsSuccess()) {
    std::string remoteETag = headOutcome.GetResult().GetETag();
    std::cout << "ETag from S3: " << remoteETag << std::endl;
}

Aws::S3::Model::PutObjectRequest request;
request.SetBucket("my-secure-bucket");
request.SetKey("encrypted_file.txt");

// 암호화 설정 (AES-256)
request.SetServerSideEncryption(Aws::S3::Model::ServerSideEncryption::AES256);

Aws::S3::Model::PutObjectRequest request;
request.SetBucket("my-secure-bucket");
request.SetKey("kms_encrypted_file.txt");

// AWS KMS 키 사용 (KMS 키 ID 필요)
request.SetServerSideEncryption(Aws::S3::Model::ServerSideEncryption::aws_kms);
request.SetSSEKMSKeyId("arn:aws:kms:us-east-1:123456789012:key/my-kms-key-id");

openssl enc -aes-256-cbc -salt -in sensitive_data.txt -out encrypted_data.txt -k mypassword

#include <openssl/evp.h>
#include <openssl/rand.h>
#include <iostream>
#include <fstream>

void EncryptAES(const std::string& inputFile, const std::string& outputFile, const std::string& key) {
    std::ifstream inFile(inputFile, std::ios::binary);
    std::ofstream outFile(outputFile, std::ios::binary);

    // AES-256-CBC 암호화 설정
    unsigned char iv[16];
    RAND_bytes(iv, sizeof(iv));

    EVP_CIPHER_CTX* ctx = EVP_CIPHER_CTX_new();
    EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), nullptr, (unsigned char*)key.c_str(), iv);

    // 암호화 처리...

    EVP_CIPHER_CTX_free(ctx);
}

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": "s3:PutObject",
            "Resource": "arn:aws:s3:::my-secure-bucket/*",
            "Condition": {
                "StringEquals": {
                    "aws:username": "trusted-user"
                }
            }
        }
    ]
}

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Deny",
            "Principal": "*",
            "Action": "s3:PutObject",
            "Resource": "arn:aws:s3:::my-secure-bucket/*",
            "Condition": {
                "StringNotEquals": {
                    "s3:x-amz-server-side-encryption": "AES256"
                }
            }
        }
    ]
}

aws ec2 create-vpc-endpoint \
    --vpc-id vpc-12345678 \
    --service-name com.amazonaws.us-east-1.s3 \
    --route-table-ids rtb-12345678

{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Deny",
            "Principal": "*",
            "Action": "s3:*",
            "Resource": "arn:aws:s3:::my-secure-bucket/*",
            "Condition": {
                "StringNotEquals": {
                    "aws:SourceVpc": "vpc-12345678"
                }
            }
        }
    ]
}