We should be POSTing files to the load endpoint

.terraform.lock.hcl

@@ -1,6 +1,25 @@
 # This file is maintained automatically by "terraform init".
 # Manual edits may be lost in future updates.
 
+provider "registry.terraform.io/hashicorp/archive" {
+  version = "2.7.1"
+  hashes = [
+    "h1:62VrkalDPMKB9zerCBS4iKTbvxejwnAWn/XXYZZQWD4=",
+    "zh:19881bb356a4a656a865f48aee70c0b8a03c35951b7799b6113883f67f196e8e",
+    "zh:2fcfbf6318dd514863268b09bbe19bfc958339c636bcbcc3664b45f2b8bf5cc6",
+    "zh:3323ab9a504ce0a115c28e64d0739369fe85151291a2ce480d51ccbb0c381ac5",
+    "zh:362674746fb3da3ab9bd4e70c75a3cdd9801a6cf258991102e2c46669cf68e19",
+    "zh:7140a46d748fdd12212161445c46bbbf30a3f4586c6ac97dd497f0c2565fe949",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:875e6ce78b10f73b1efc849bfcc7af3a28c83a52f878f503bb22776f71d79521",
+    "zh:b872c6ed24e38428d817ebfb214da69ea7eefc2c38e5a774db2ccd58e54d3a22",
+    "zh:cd6a44f731c1633ae5d37662af86e7b01ae4c96eb8b04144255824c3f350392d",
+    "zh:e0600f5e8da12710b0c52d6df0ba147a5486427c1a2cc78f31eea37a47ee1b07",
+    "zh:f21b2e2563bbb1e44e73557bcd6cdbc1ceb369d471049c40eb56cb84b6317a60",
+    "zh:f752829eba1cc04a479cf7ae7271526b402e206d5bcf1fcce9f535de5ff9e4e6",
+  ]
+}
+
 provider "registry.terraform.io/hashicorp/aws" {
   version = "5.99.1"
   hashes = [

README.md

@@ -6,13 +6,22 @@
 - [ ] Terraform repo scaffolding out the service
 ## Air Traffic Metrics
 - [ ] Ingest raw data into S3
-    - [ ] Bucket best practices
+    - [X] S3 Bucket
-        - [ ] Buckets aren't public
+        - [ ] Bucket best practices
-        - [ ]
+            - [X] Buckets aren't public
+            - [X] Only `load` lambda can add objects
+            - [X] Only Athena can read objects
+            - [ ] ????
 - [ ] Database of my choice
-    - [ ] Athena?
+    - [X] Athena?
+        - Less up-front configuration involved. I hope.
+        - AWS wants to lock you into their ecosystem so it'll be cheaper.
+        - In the future, we can switch from CSV to JSON or Parquet or mix and match without any overhead.
     - [ ] Aurora (Postgresql)
+    - [ ] Aurora (Mysql)
 - [ ] API to interact with the data
+    - [ ] API Gateway
+        - [ ] Policy that grants it lambda invocation rights
     - [ ] `/load` endpoint
         - [ ] Python lambda to upload data to bucket
             - [ ] Example data: https://zenodo.org/records/5377831
@@ -22,5 +31,6 @@
             - [ ] last_transponder_seen_at: maximum value for `lastseen`
             - [ ] most_popular_destination: most seen value for destination
             - [ ] count_of_unique_transponders: unique count of the icao24 field
+        - [ ] Execution role that grants it access to athena database
 - [ ] Frontend to display the data
     - [ ] Python lambda that renders an HTML page that displays above metrics

api_gw.load.tf

@@ -0,0 +1,30 @@
+resource "aws_api_gateway_resource" "transponder-tracker-load" {
+  parent_id   = aws_api_gateway_rest_api.transponder-tracker.root_resource_id
+  path_part   = "load"
+  rest_api_id = aws_api_gateway_rest_api.transponder-tracker.id
+}
+
+resource "aws_api_gateway_method" "transponder-tracker-load" {
+  authorization = "NONE"
+  http_method   = "POST"
+  resource_id   = aws_api_gateway_resource.transponder-tracker-load.id
+  rest_api_id   = aws_api_gateway_rest_api.transponder-tracker.id
+}
+
+resource "aws_api_gateway_integration" "transponder-tracker-load" {
+  http_method = aws_api_gateway_method.transponder-tracker-load.http_method
+  resource_id = aws_api_gateway_resource.transponder-tracker-load.id
+  rest_api_id = aws_api_gateway_rest_api.transponder-tracker.id
+  type        = "AWS_PROXY"
+  uri         = module.load-lambda.invoke_arn
+}
+
+resource "aws_lambda_permission" "apigw_lambda_load" {
+  statement_id  = "AllowExecutionFromAPIGateway"
+  action        = "lambda:InvokeFunction"
+  function_name = module.load-lambda.function_name
+  principal     = "apigateway.amazonaws.com"
+
+  # More: http://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-control-access-using-iam-policies-to-invoke-api.html
+  source_arn = "arn:aws:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.id}:${aws_api_gateway_rest_api.transponder-tracker.id}/*/${aws_api_gateway_method.transponder-tracker-load.http_method}${aws_api_gateway_resource.transponder-tracker-load.path}"
+}

api_gw.metrics.tf

@@ -0,0 +1,30 @@
+resource "aws_api_gateway_resource" "transponder-tracker-metrics" {
+  parent_id   = aws_api_gateway_rest_api.transponder-tracker.root_resource_id
+  path_part   = "metrics"
+  rest_api_id = aws_api_gateway_rest_api.transponder-tracker.id
+}
+
+resource "aws_api_gateway_method" "transponder-tracker-metrics" {
+  authorization = "NONE"
+  http_method   = "GET"
+  resource_id   = aws_api_gateway_resource.transponder-tracker-metrics.id
+  rest_api_id   = aws_api_gateway_rest_api.transponder-tracker.id
+}
+
+resource "aws_api_gateway_integration" "transponder-tracker-metrics" {
+  http_method = aws_api_gateway_method.transponder-tracker-metrics.http_method
+  resource_id = aws_api_gateway_resource.transponder-tracker-metrics.id
+  rest_api_id = aws_api_gateway_rest_api.transponder-tracker.id
+  type        = "AWS_PROXY"
+  uri         = module.metrics-lambda.invoke_arn
+}
+
+resource "aws_lambda_permission" "apigw_lambda-metrics" {
+  statement_id  = "AllowExecutionFromAPIGateway"
+  action        = "lambda:InvokeFunction"
+  function_name = module.metrics-lambda.function_name
+  principal     = "apigateway.amazonaws.com"
+
+  # More: http://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-control-access-using-iam-policies-to-invoke-api.html
+  source_arn = "arn:aws:execute-api:${data.aws_region.current.name}:${data.aws_caller_identity.current.id}:${aws_api_gateway_rest_api.transponder-tracker.id}/*/${aws_api_gateway_method.transponder-tracker-metrics.http_method}${aws_api_gateway_resource.transponder-tracker-metrics.path}"
+}

api_gw.tf

@@ -0,0 +1,32 @@
+resource "aws_api_gateway_rest_api" "transponder-tracker" {
+  name = "transponder-tracker"
+}
+
+resource "aws_api_gateway_deployment" "transponder-tracker" {
+  rest_api_id = aws_api_gateway_rest_api.transponder-tracker.id
+
+  triggers = {
+    # NOTE: The configuration below will satisfy ordering considerations,
+    # but not pick up all future REST API changes. More advanced patterns
+    # are possible, such as using the filesha1() function against the
+    # Terraform configuration file(s) or removing the .id references to
+    # calculate a hash against whole resources. Be aware that using whole
+    # resources will show a difference after the initial implementation.
+    # It will stabilize to only change when resources change afterwards.
+    redeployment = sha1(jsonencode([
+      aws_api_gateway_resource.transponder-tracker-load.id,
+      aws_api_gateway_method.transponder-tracker-load.id,
+      aws_api_gateway_integration.transponder-tracker-load.id,
+    ]))
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+resource "aws_api_gateway_stage" "transponder-tracker" {
+  deployment_id = aws_api_gateway_deployment.transponder-tracker.id
+  rest_api_id   = aws_api_gateway_rest_api.transponder-tracker.id
+  stage_name    = "sandbox"
+}

frontend.tf

@@ -0,0 +1,17 @@
+data "aws_iam_policy_document" "lambda-frontend-role" {
+  statement {
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+    actions = ["sts:AssumeRole"]
+  }
+}
+
+module "frontend-lambda" {
+  source  = "./modules/lambda"
+  name    = "frontend"
+  handler = "frontend"
+  policy  = data.aws_iam_policy_document.lambda-frontend-role.json
+}

load.tf

@@ -9,25 +9,9 @@ data "aws_iam_policy_document" "lambda-load-role" {
   }
 }
 
-resource "aws_iam_role" "lambda-load-role" {
+module "load-lambda" {
-  name               = "lambda-load-role"
+  source  = "./modules/lambda"
-  assume_role_policy = data.aws_iam_policy_document.lambda-load-role.json
+  name    = "load"
+  handler = "load"
+  policy  = data.aws_iam_policy_document.lambda-load-role.json
 }
-
-data "archive_file" "lambda-load-role" {
-  type        = "zip"
-  source_file = "src/load.py"
-  output_path = "src/load.zip"
-}
-
-resource "aws_lambda_function" "load" {
-  function_name = "transponder-tracker-load"
-  runtime       = "python3.11"
-
-  role = aws_iam_role.lambda-load-role.arn
-
-  handler          = "load"
-  filename         = "src/load.zip"
-  source_code_hash = data.archive_file.lambda-load-role.output_base64sha256
-}
-

main.tf

@@ -0,0 +1,3 @@
+data "aws_caller_identity" "current" {}
+
+data "aws_region" "current" {}

metrics.tf

@@ -9,25 +9,9 @@ data "aws_iam_policy_document" "lambda-metrics-role" {
   }
 }
 
-resource "aws_iam_role" "lambda-metrics-role" {
+module "metrics-lambda" {
-  name               = "lambda-metrics-role"
+  source  = "./modules/lambda"
-  assume_role_policy = data.aws_iam_policy_document.lambda-metrics-role.json
+  name    = "metrics"
+  handler = "metrics"
+  policy  = data.aws_iam_policy_document.lambda-metrics-role.json
 }
-
-data "archive_file" "lambda-metrics-role" {
-  type        = "zip"
-  source_file = "src/metrics.py"
-  output_path = "src/metrics.zip"
-}
-
-resource "aws_lambda_function" "metrics" {
-  function_name = "transponder-tracker-metrics"
-  runtime       = "python3.11"
-
-  role = aws_iam_role.lambda-metrics-role.arn
-
-  handler          = "metrics"
-  filename         = "src/metrics.zip"
-  source_code_hash = data.archive_file.lambda-metrics-role.output_base64sha256
-}
-

modules/lambda/lambda.tf

@@ -0,0 +1,22 @@
+resource "aws_iam_role" "lambda-role" {
+  name               = "lambda-role"
+  assume_role_policy = var.policy
+}
+
+data "archive_file" "lambda-role" {
+  type        = "zip"
+  source_file = "src/${var.name}.py"
+  output_path = "src/${var.name}.zip"
+}
+
+resource "aws_lambda_function" "lambda" {
+  function_name = var.name
+  runtime       = "python3.11"
+
+  role = aws_iam_role.lambda-role.arn
+
+  handler          = var.handler
+  filename         = "src/${var.name}.zip"
+  source_code_hash = data.archive_file.lambda-role.output_base64sha256
+}
+

modules/lambda/outputs.tf

@@ -0,0 +1,11 @@
+output "arn" {
+  value = aws_lambda_function.lambda.arn
+}
+
+output "invoke_arn" {
+  value = aws_lambda_function.lambda.invoke_arn
+}
+
+output "function_name" {
+  value = aws_lambda_function.lambda.function_name
+}

modules/lambda/variables.tf

@@ -0,0 +1,11 @@
+variable "name" {
+  type = string
+}
+
+variable "handler" {
+  type = string
+}
+
+variable "policy" {
+  type = string
+}

s3.tf

@@ -9,3 +9,42 @@ resource "aws_s3_bucket_versioning" "transponder-data-versioning" {
   }
 }
 
+data "aws_iam_policy_document" "transponder-data-policy" {
+  statement {
+    sid    = "athena-read"
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["athena.amazonaws.com"]
+    }
+    actions = [
+      "s3:GetObject",
+      "s3:ListBucket",
+    ]
+    resources = [
+      aws_s3_bucket.transponder-data.arn,
+      "${aws_s3_bucket.transponder-data.arn}/*",
+    ]
+  }
+
+  statement {
+    sid    = "lambda-write"
+    effect = "Allow"
+    principals {
+      type        = "AWS"
+      identifiers = [module.metrics-lambda.arn]
+    }
+    actions = [
+      "s3:PutObject",
+    ]
+    resources = [
+      aws_s3_bucket.transponder-data.arn,
+      "${aws_s3_bucket.transponder-data.arn}/*",
+    ]
+  }
+}
+
+resource "aws_s3_bucket_policy" "transponder-data" {
+  bucket = aws_s3_bucket.transponder-data.id
+  policy = data.aws_iam_policy_document.transponder-data-policy.json
+}

src/frontend.py

@@ -0,0 +1 @@
+pass