import json import boto3 import s3_backup class Downstream: """Push message to downstream""" def __init__(self): self.client = boto3.client("sqs") self.queues = {} def push(self, dedup_id, group_id, body): try: name = "{}-mim.fifo".format(group_id) if name not in self.queues: # mark the queue not exist first, then try to get the queue url # if success, overwrite the queue url self.queues[name] = None queue = self.client.get_queue_url(QueueName=name) self.queues[name] = queue['QueueUrl'] if self.queues[name]: self.client.send_message( QueueUrl=self.queues[name], MessageGroupId=group_id, MessageBody=body, MessageDeduplicationId=dedup_id) else: print("the queue may not exist: {0}, ignoring this message...".format(group_id)) except Exception as err: print("error on push record to downstream({0}): {1}".format(group_id, err)) def message_transformers(): return [ # disable s3 backup since we will use kafka as the stable chanel # s3_backup.transformer(), ] def lambda_handler(event, context): downstream = Downstream() transformers = message_transformers() for record in event["Records"]: dedup_id = record["attributes"]["MessageDeduplicationId"] group_id = record["attributes"]["MessageGroupId"] body = record["body"] for transformer in transformers: try: if transformer.match(group_id): body = transformer.transform(group_id, body) except Exception as err: print("error on handle record({0}): {1}", json.dumps(record), err) downstream.push(dedup_id, group_id, body)