serverless.yml

service: sls-data-pipelines

package:
  individually: true

plugins:
  - serverless-iam-roles-per-function
  - serverless-python-requirements
  - serverless-pseudo-parameters


custom:
  lumigo:
    token: ${ssm:/lumigo/token}
  firehoseS3Prefix: 'destination/'
  firehoseS3AnylticsResultsPrefix: 'analytics/results/'
  firehoseCompressionFormat: UNCOMPRESSED
  stage: "${opt:stage, self:provider.stage}"

provider:
  name: aws
  stage: dev
  runtime: python3.8
  region: eu-west-1
  profile: default
  httpApi:
    cors:
      allowedOrigins:
        - '*'
      allowedMethods:
        - GET
        - OPTIONS
  reservedConcurrency: 1
  memorySize: 512
  versionFunctions: false
  timeout: 30
  iamRoleStatements:
    - Effect: "Allow"
      Action:
        - "logs:CreateLogGroup"
        - "logs:CreateLogStream"
        - "logs:PutLogEvents"
      Resource: '*'
  environment:
    LUMIGO_TRACER_TOKEN: ${ssm:/lumigo/token}
    LUMIGO_ORIGINAL_HANDLER: handler.handle

functions:
  RetrieveXMLTrafficData:
    handler: /opt/python/lumigo_tracer._handler
    module: src/RetrieveXMLTrafficData
    timeout: 60
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      TRAFFIC_DATA_URL: "http://miv.opendata.belfla.be/miv/verkeersdata"
      BUCKET_NAME:
        Ref: OriginBucket
      S3_XML_PREFIX: "xml/input/"
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - s3:PutObject
        Resource:
          Fn::Join:
            - ""
            - - "arn:aws:s3:::"
              - Ref: "OriginBucket"
              - "/*"
    events:
      - schedule:
          rate: rate(1 minute)
          enabled: false

  TransformTrafficToJsonData:
    handler: /opt/python/lumigo_tracer._handler
    module: src/TransformTrafficDataToJson
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      BUCKET_NAME:
        Ref: OriginBucket
      S3_XML_PREFIX: "xml/input/"
      S3_JSON_PREFIX: "json/input/"
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - s3:GetObject
          - s3:PutObject
        Resource:
          Fn::Join:
            - ""
            - - "arn:aws:s3:::"
              - Ref: "OriginBucket"
              - "/*"
    events:
      - s3:
          bucket:
            Ref: OriginBucket
          event: s3:ObjectCreated:*
          rules:
            - prefix: xml/input/
            - suffix: .xml
          existing: true

  PublishTrafficData:
    handler: /opt/python/lumigo_tracer._handler
    module: src/PublishTrafficData
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      BUCKET_NAME:
        Ref: OriginBucket
      S3_JSON_PREFIX: "json/input/"
      DELIVERY_STREAM_NAME:
        Ref: DeliveryStream
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - s3:GetObject
        Resource:
          Fn::Join:
            - ""
            - - "arn:aws:s3:::"
              - Ref: "OriginBucket"
              - "/*"
      - Effect: "Allow"
        Action:
          - "firehose:PutRecord"
          - "firehose:PutRecordBatch"
        Resource: { Fn::GetAtt: [ DeliveryStream, Arn ] }
    events:
      - s3:
          bucket:
            Ref: OriginBucket
          event: s3:ObjectCreated:*
          existing: true
          rules:
            - prefix: json/input/
            - suffix: .json

  StreamProcessor:
    handler: /opt/python/lumigo_tracer._handler
    module: src/StreamProcessor
    timeout: 300
    environment:
      FILTER_IDS: '1897, 3159, 3977, 1065, 575'
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - "firehose:PutRecord"
          - "firehose:PutRecordBatch"
        Resource: !GetAtt DeliveryStream.Arn

  StoreAnalyticsResultsPerPoint:
    handler: /opt/python/lumigo_tracer._handler
    module: src/StoreAnalyticsResultsPerPoint
    timeout: 10
    reservedConcurrency: 1
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      TABLE_NAME: !Ref RealTimeAnalyticsPerPointTable
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - "kinesis:DescribeStream"
          - "kinesis:GetShardIterator"
          - "kinesis:GetRecords"
        Resource: !GetAtt DestinationStream.Arn
      - Effect: "Allow"
        Action:
          - "dynamodb:DescribeStream"
          - "dynamodb:DescribeTable"
          - "dynamodb:BatchWrite*"
          - "dynamodb:Delete*"
          - "dynamodb:Update*"
          - "dynamodb:PutItem"
        Resource: !GetAtt RealTimeAnalyticsPerPointTable.Arn
      - Effect: "Allow"
        Action:
          - "sqs:SendMessage"
          - "sqs:ListQueues"
        Resource: !GetAtt SaveAnalyticsDeadQueue.Arn
    events:
      - stream:
          type: kinesis
          arn: !GetAtt DestinationStream.Arn
          batchWindow: 10
          batchSize: 20
          maximumRetryAttempts: 2
          startingPosition: LATEST
          enabled: true
          destinations:
            onFailure:
              arn: !GetAtt SaveAnalyticsDeadQueue.Arn
              type: sqs
  PowerBiForwarderr:
#    handler: /opt/python/lumigo_tracer._handler
    handler: handler.handle
    module: src/PowerBiForwarder
    timeout: 10
    reservedConcurrency: 1
#    layers:
#      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      STEAMING_DATASET_URL: https://api.powerbi.com/beta/49c3d703-3579-47bf-a888-7c913fbdced9/datasets/a52e28e9-93f2-4c07-934f-52818e43536d/rows?noSignUpCheck=1&key=aYKSacZqpKWp7w1kUNI6KQJNejVi95yZkv%2FHJQDtLuZFrI3%2FhQkjYdoHsT9%2BoYSShdMZWQevrf5i77sdQHgYUA%3D%3D
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - "kinesis:DescribeStream"
          - "kinesis:GetShardIterator"
          - "kinesis:GetRecords"
        Resource: !GetAtt DestinationStream.Arn
      - Effect: "Allow"
        Action:
          - "sqs:SendMessage"
          - "sqs:ListQueues"
        Resource: !GetAtt PowerBiForwarderDeadQueue.Arn
    events:
      - stream:
          type: kinesis
          arn: !GetAtt DestinationStream.Arn
          batchWindow: 10
          batchSize: 20
          maximumRetryAttempts: 2
          startingPosition: LATEST
          enabled: true
          destinations:
            onFailure:
              arn: !GetAtt PowerBiForwarderDeadQueue.Arn
              type: sqs

  AlertSaver:
    handler: /opt/python/lumigo_tracer._handler
    module: src/AlertSaver
    timeout: 10
    reservedConcurrency: 1
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      TABLE_NAME: !Ref TrafficJamAlertsTable
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - "dynamodb:DescribeStream"
          - "dynamodb:DescribeTable"
          - "dynamodb:BatchWrite*"
          - "dynamodb:Delete*"
          - "dynamodb:Update*"
          - "dynamodb:PutItem"
        Resource: !GetAtt RealTimeAnalyticsPerPointTable.Arn
      - Effect: "Allow"
        Action:
          - "dynamodb:DescribeStream"
          - "dynamodb:DescribeTable"
          - "dynamodb:BatchWrite*"
          - "dynamodb:Delete*"
          - "dynamodb:Update*"
          - "dynamodb:PutItem"
        Resource: !GetAtt TrafficJamAlertsTable.Arn
    events:
      - stream:
          type: dynamodb
          arn:
            Fn::GetAtt: [RealTimeAnalyticsPerPointTable, StreamArn]
          maximumRetryAttempts: 2
  AlertSender:
    handler: /opt/python/lumigo_tracer._handler
    module: src/AlertSender
    timeout: 10
    reservedConcurrency: 1
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      SLACK_WEBHOOK: ${ssm:/slack/traffic/webhook}
      SLACK_CHANNEL: traffic-alerts
      FLASK_API: http://flask-env.eba-jp3gn4xp.eu-west-1.elasticbeanstalk.com/send-notification
    iamRoleStatementsInherit: true
    events:
      - stream:
          type: dynamodb
          arn:
            Fn::GetAtt: [TrafficJamAlertsTable, StreamArn]
          maximumRetryAttempts: 2
  CurrentSituation:
    handler: /opt/python/lumigo_tracer._handler
    module: src/CurrentSituation
    timeout: 10
    reservedConcurrency: 1
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      TABLE_NAME: !Ref RealTimeAnalyticsPerPointTable
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - "dynamodb:BatchGet*"
          - "dynamodb:DescribeStream"
          - "dynamodb:DescribeTable"
          - "dynamodb:Get*"
          - "dynamodb:Query"
          - "dynamodb:Scan"
        Resource: !GetAtt RealTimeAnalyticsPerPointTable.Arn
    events:
      - httpApi:
          method: GET
          path: /traffic
  CurrentAlerts:
    handler: /opt/python/lumigo_tracer._handler
    module: src/CurrentAlerts
    timeout: 10
    reservedConcurrency: 1
    layers:
      - arn:aws:lambda:eu-west-1:114300393969:layer:lumigo-python-tracer:115
    environment:
      TABLE_NAME: !Ref TrafficJamAlertsTable
    iamRoleStatementsInherit: true
    iamRoleStatements:
      - Effect: "Allow"
        Action:
          - "dynamodb:BatchGet*"
          - "dynamodb:DescribeStream"
          - "dynamodb:DescribeTable"
          - "dynamodb:Get*"
          - "dynamodb:Query"
          - "dynamodb:Scan"
        Resource: !GetAtt TrafficJamAlertsTable.Arn
    events:
      - httpApi:
          method: GET
          path: /alerts
resources:
  - ${file(resources/firehose.yml)}
  - ${file(resources/kinesis.yml)}
  - ${file(resources/sqs.yml)}
  - ${file(resources/s3.yml)}
  - ${file(resources/dynamodb.yml)}
  - ${file(resources/analytics.yml)}

  - Outputs:
      DeliveryStreamName:
        Description: "Name of the firehose over which all traffic events flow"
        Value: !Ref DeliveryStream
      RealTimeAnalyticsPerPointTableName:
        Description: "Single table that holds the analytics results"
        Value: !Ref RealTimeAnalyticsPerPointTable
      TrafficJamAlertsTableName:
        Description: "Table that holds the traffic jam alerts"
        Value: !Ref TrafficJamAlertsTable
      OriginBucketName:
        Description: "Bucket where input files are placed"
        Value: !Ref OriginBucket
      ResourcesBucketName:
        Description: "Bucket where config files are placed"
        Value: !Ref ResourcesBucket
      TrafficEventsDeliveryBucket:
        Value: !Ref DeliveryBucket
        Export:
          Name: ${self:service}-${opt:stage, self:provider.stage, 'dev'}-TrafficEventsDeliveryBucketName