Add a terraform example to deploy Quickwit lambdas

[fmassot]

Coming from a redditor: provide terraform examples to deploy Quickwit Lambdas.

[kalil-pelissier]

Hi 👋, I will be happy to start working on this issue if it's still up to date!

[bjernie]

@kalil-pelissier Do you have any update?

[kalil-pelissier]

Hi, @bjernie didn't start to work on it. Do you want to work on this issue?

[bjernie]

No, not right now

[hjander]

Hi @kalil-pelissier , i would like to try. Any hints or pointers ?

[bjernie]

@hjander I decided to give I at try and it worked beautifully. I am not yet ready to create a PR but this is the terraform code.

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = ">=5.51.1"
    }
  }
}

provider "aws" {
  region = "eu-west-3"
}

locals {
  index_config_key = "index-config.yaml"
  index_config = yamldecode(file("../${local.index_config_key}"))
}

// S3
resource "aws_s3_bucket" "index" {
  bucket        = "quickwit-index-bucket"
  force_destroy = true
}

resource "aws_s3_bucket_policy" "index" {
  bucket = aws_s3_bucket.index.bucket
  policy = data.aws_iam_policy_document.index_policy.json
}

data "aws_iam_policy_document" "index_policy" {
  statement {
    effect  = "Allow"
    actions = ["s3:*"]
    resources = [
      aws_s3_bucket.index.arn,
      "${aws_s3_bucket.index.arn}/*"
    ]
    principals {
      type = "Service"
      identifiers = [
        "lambda.amazonaws.com"
      ]
    }
  }
}

// Upload the index config to the bucket
resource "aws_s3_object" "index_config" {
  bucket = aws_s3_bucket.index.bucket
  key    = local.index_config_key
  content = file("../${local.index_config_key}")
}

// Indexer Lambda
module "indexer_lambda" {
  source             = "terraform-aws-modules/lambda/aws"
  function_name      = "quickwit-indexer"
  source_path        = "cdk.out/indexer/bootstrap"
  handler            = "bootstrap"
  runtime            = "provided.al2023"
  memory_size        = 3008
  timeout            = 900
  attach_policy_json = true
  policy_json        = data.aws_iam_policy_document.indexer_lambda_policy.json
  environment_variables = {
    QW_LAMBDA_INDEX_BUCKET     = aws_s3_bucket.index.bucket
    QW_LAMBDA_METASTORE_BUCKET = aws_s3_bucket.index.bucket
    QW_LAMBDA_INDEX_ID         = local.index_config.index_id
    QW_LAMBDA_INDEX_CONFIG_URI = "s3://${aws_s3_bucket.index.bucket}/${local.index_config_key}"
    RUST_LOG                   = "quickwit=debug"
  }
}

data "aws_iam_policy_document" "indexer_lambda_policy" {
  statement {
    effect = "Allow"
    actions = [
      "s3:*"
    ]
    resources = [
      aws_s3_bucket.index.arn,
      "${aws_s3_bucket.index.arn}/*"
    ]
  }
}

// Searcher Lambda
module "searcher_lambda" {
  source             = "terraform-aws-modules/lambda/aws"
  function_name      = "quickwit-searcher"
  source_path        = "cdk.out/searcher/bootstrap"
  handler            = "bootstrap"
  runtime            = "provided.al2023"
  memory_size        = 3008
  timeout            = 30
  attach_policy_json = true
  policy_json        = data.aws_iam_policy_document.searcher_lambda_policy.json
  environment_variables = {
    QW_LAMBDA_INDEX_BUCKET     = aws_s3_bucket.index.bucket
    QW_LAMBDA_METASTORE_BUCKET = aws_s3_bucket.index.bucket
    QW_LAMBDA_INDEX_ID         = local.index_config.index_id
    RUST_LOG                   = "quickwit=debug"
  }
}

data "aws_iam_policy_document" "searcher_lambda_policy" {
  statement {
    effect = "Allow"
    actions = [
      "s3:*"
    ]
    resources = [
      aws_s3_bucket.index.arn,
      "${aws_s3_bucket.index.arn}/*"
    ]
  }
  statement {
    effect = "Allow"
    actions = [
      "s3:GetObject"
    ]
    resources = ["arn:aws:s3:::quickwit-datasets-public/*"]
  }
  statement {
    effect = "Allow"
    actions = [
      "s3:GetObject"
    ]
    resources = [
      aws_s3_bucket.index.arn,
      "${aws_s3_bucket.index.arn}/*",
    ]
  }
}

This Terraform example is based on the lambda beta 01, which works like as its supposed to. But trying to use the latest beta 04 makes the searcher lambda throw a "route not found" error, which I haven't managed to find a fix for yet.