From 3aef679eb22a989b1aa08f31d4b068cddb85821c Mon Sep 17 00:00:00 2001 From: guyrenny Date: Mon, 24 Jun 2024 10:54:46 +0300 Subject: [PATCH 1/2] update lambda-manager code to trigger on creation --- src/lambda-manager/CHANGELOG.md | 4 + src/lambda-manager/lambda_function.py | 147 +++++++++++++++----------- src/lambda-manager/requirements.txt | 1 + src/lambda-manager/template.yaml | 14 ++- 4 files changed, 101 insertions(+), 65 deletions(-) create mode 100644 src/lambda-manager/requirements.txt diff --git a/src/lambda-manager/CHANGELOG.md b/src/lambda-manager/CHANGELOG.md index ef75337..ea3a313 100644 --- a/src/lambda-manager/CHANGELOG.md +++ b/src/lambda-manager/CHANGELOG.md @@ -2,6 +2,10 @@ ## lambda-manager +## 2.0.2 / 24-06-2024 +### 💡 Enhancements 💡 +- Update the lambda to trigger on creation if ScanOldLogGroups is set to true + ## 2.0.1 / 04-2-2024 ### 💡 Enhancements 💡 - Update lambda code so it will not require the allow all policy diff --git a/src/lambda-manager/lambda_function.py b/src/lambda-manager/lambda_function.py index 4711c48..c499001 100644 --- a/src/lambda-manager/lambda_function.py +++ b/src/lambda-manager/lambda_function.py @@ -2,11 +2,13 @@ import os import re import uuid +import cfnresponse + def identify_arn_service(arn): - arn_parts = arn.split(':') + arn_parts = arn.split(':') if len(arn_parts) < 6: - return "Invalid ARN format" - service = arn_parts[2] + return "Invalid ARN format" + service = arn_parts[2] if service == "lambda": return "lambda" elif service == "firehose": @@ -25,10 +27,10 @@ def list_log_groups_and_subscriptions(cloudwatch_logs, regex_pattern, logs_filte response = cloudwatch_logs.describe_log_groups(**kwargs) log_groups.extend(response['logGroups']) for log_group in log_groups: - log_group_name = log_group['logGroupName'] + log_group_name = log_group['logGroupName'] if regex_pattern and re.match(regex_pattern, log_group_name): print(f"Log Group: {log_group_name}") - + subscriptions = cloudwatch_logs.describe_subscription_filters(logGroupName=log_group_name) subscriptions = subscriptions.get('subscriptionFilters') print(f" Subscriptions: {subscriptions} for {log_group_name}") @@ -121,75 +123,92 @@ def list_log_groups_and_subscriptions(cloudwatch_logs, regex_pattern, logs_filte else: print(f"Invalid destination type {destination_type}") - - def lambda_handler(event, context): - cloudwatch_logs = boto3.client('logs') - regex_pattern = os.environ.get('REGEX_PATTERN') - destination_type = os.environ.get('DESTINATION_TYPE') - logs_filter = os.environ.get('LOGS_FILTER', '') - scan_all_log_groups = os.environ.get('SCAN_OLD_LOGGROUPS', 'false') - destination_arn = os.environ.get('DESTINATION_ARN') - role_arn = os.environ.get('DESTINATION_ROLE') - filter_name = 'Coralogix_Filter_' + str(uuid.uuid4()) - print(f"Scanning all log groups: {scan_all_log_groups}") - if scan_all_log_groups == 'true': - list_log_groups_and_subscriptions(cloudwatch_logs, regex_pattern, logs_filter, destination_arn, role_arn, filter_name, context) - lambda_client = boto3.client('lambda') - function_name = context.function_name + status = cfnresponse.SUCCESS + try: + cloudwatch_logs = boto3.client('logs') + regex_pattern = os.environ.get('REGEX_PATTERN') + destination_type = os.environ.get('DESTINATION_TYPE') + logs_filter = os.environ.get('LOGS_FILTER', '') + scan_all_log_groups = os.environ.get('SCAN_OLD_LOGGROUPS', 'false') + destination_arn = os.environ.get('DESTINATION_ARN') + role_arn = os.environ.get('DESTINATION_ROLE') + filter_name = 'Coralogix_Filter_' + str(uuid.uuid4()) - # Fetch the current function configuration - current_config = lambda_client.get_function_configuration(FunctionName=function_name) - current_env_vars = current_config['Environment']['Variables'] + print(f"Scanning all log groups: {scan_all_log_groups}") + if scan_all_log_groups == 'true' and "RequestType" in event: + list_log_groups_and_subscriptions(cloudwatch_logs, regex_pattern, logs_filter, destination_arn, role_arn, filter_name, context) + lambda_client = boto3.client('lambda') + function_name = context.function_name - # Update the environment variables - current_env_vars['SCAN_OLD_LOGGROUPS'] = 'false' + # Fetch the current function configuration + current_config = lambda_client.get_function_configuration(FunctionName=function_name) + current_env_vars = current_config['Environment']['Variables'] - # Update the Lambda function configuration - try: - response = lambda_client.update_function_configuration( - FunctionName=function_name, - Environment={'Variables': current_env_vars} - ) - print("Updated environment variables:", response['Environment']['Variables']) - except Exception as e: - print("Error updating function configuration:", e) - else: - log_group_to_subscribe = event['detail']['requestParameters']['logGroupName'] - print(f"Log Group: {log_group_to_subscribe}") - if regex_pattern and re.match(regex_pattern, log_group_to_subscribe): - if destination_type == 'firehose': + # Update the environment variables + current_env_vars['SCAN_OLD_LOGGROUPS'] = 'false' + + # Update the Lambda function configuration + try: + response = lambda_client.update_function_configuration( + FunctionName=function_name, + Environment={'Variables': current_env_vars} + ) + print("Updated environment variables:", response['Environment']['Variables']) + except Exception as e: + print("Error updating function configuration:", e) + status = cfnresponse.FAILED + elif scan_all_log_groups != 'true' and "RequestType" not in event: + log_group_to_subscribe = event['detail']['requestParameters']['logGroupName'] + print(f"Log Group: {log_group_to_subscribe}") + if regex_pattern and re.match(regex_pattern, log_group_to_subscribe): + if destination_type == 'firehose': + try: + cloudwatch_logs.put_subscription_filter( + destinationArn=destination_arn, + roleArn=role_arn, + filterName= filter_name, + filterPattern=logs_filter, + logGroupName=log_group_to_subscribe, + ) + except Exception as e: + print(f"Failed to put subscription filter for {log_group_to_subscribe}: {e}") + status = cfnresponse.FAILED + elif destination_type == 'lambda': try: + lambda_client = boto3.client('lambda') + region = context.invoked_function_arn.split(":")[3] + account_id = context.invoked_function_arn.split(":")[4] + lambda_client.add_permission( + FunctionName=destination_arn, + StatementId=f'allow-trigger-from-{log_group_to_subscribe}', + Action='lambda:InvokeFunction', + Principal='logs.amazonaws.com', + SourceArn=f'arn:aws:logs:{region}:{account_id}:log-group:{log_group_to_subscribe}:*', + ) cloudwatch_logs.put_subscription_filter( destinationArn=destination_arn, - roleArn=role_arn, - filterName= filter_name, + filterName= "coralogix-aws-shipper-cloudwatch-trigger", filterPattern=logs_filter, logGroupName=log_group_to_subscribe, ) except Exception as e: print(f"Failed to put subscription filter for {log_group_to_subscribe}: {e}") - elif destination_type == 'lambda': - try: - lambda_client = boto3.client('lambda') - region = context.invoked_function_arn.split(":")[3] - account_id = context.invoked_function_arn.split(":")[4] - lambda_client.add_permission( - FunctionName=destination_arn, - StatementId=f'allow-trigger-from-{log_group_to_subscribe}', - Action='lambda:InvokeFunction', - Principal='logs.amazonaws.com', - SourceArn=f'arn:aws:logs:{region}:{account_id}:log-group:{log_group_to_subscribe}:*', - ) - cloudwatch_logs.put_subscription_filter( - destinationArn=destination_arn, - filterName= "coralogix-aws-shipper-cloudwatch-trigger", - filterPattern=logs_filter, - logGroupName=log_group_to_subscribe, - ) - except Exception as e: - print(f"Failed to put subscription filter for {log_group_to_subscribe}: {e}") + status = cfnresponse.FAILED + else: + print(f"Invalid destination type {destination_type}") + status = cfnresponse.FAILED else: - print(f"Invalid destination type {destination_type}") - else: - print(f"Loggroup {log_group_to_subscribe} excluded") + print(f"Loggroup {log_group_to_subscribe} excluded") + except Exception as e: + print(f"Failed with exception: {e}") + status = cfnresponse.FAILED + finally: + print("Sending response to custom resource") + cfnresponse.send( + event, + context, + status, + {}, + event.get('PhysicalResourceId', context.aws_request_id) + ) diff --git a/src/lambda-manager/requirements.txt b/src/lambda-manager/requirements.txt new file mode 100644 index 0000000..22fdf3c --- /dev/null +++ b/src/lambda-manager/requirements.txt @@ -0,0 +1 @@ +cfnresponse diff --git a/src/lambda-manager/template.yaml b/src/lambda-manager/template.yaml index c871efa..1312c79 100644 --- a/src/lambda-manager/template.yaml +++ b/src/lambda-manager/template.yaml @@ -16,7 +16,7 @@ Metadata: - cloudwatch - lambda HomePageUrl: https://coralogix.com - SemanticVersion: 2.0.1 + SemanticVersion: 2.0.2 SourceCodeUrl: https://github.com/coralogix/coralogix-aws-serverless AWS::CloudFormation::Interface: ParameterGroups: @@ -81,6 +81,10 @@ Conditions: - Fn::Equals: - Ref: NotificationEmail - '' + ScanOldLogGroups: + Fn::Equals: + - Ref: ScanOldLogGroups + - 'true' Resources: LambdaExecutionRole: Type: AWS::IAM::Role @@ -175,6 +179,14 @@ Resources: - "CreateLogGroup" Target: Id: cx-loggroup-target + + LambdaTrigger: + Type: Custom::LambdaTrigger + Condition: ScanOldLogGroups + DependsOn: LambdaFunction + Properties: + ServiceToken: !GetAtt LambdaFunction.Arn + LambdaFunctionNotificationSubscription: Type: AWS::SNS::Subscription Condition: IsNotificationEnabled From e7f252c1a2b2e1f13d92a398f53ae040f534fdcf Mon Sep 17 00:00:00 2001 From: guyrenny Date: Mon, 24 Jun 2024 12:10:01 +0300 Subject: [PATCH 2/2] Send response to custom resource only in the creation nad deletetion of the lambda --- src/lambda-manager/lambda_function.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/lambda-manager/lambda_function.py b/src/lambda-manager/lambda_function.py index c499001..5a39507 100644 --- a/src/lambda-manager/lambda_function.py +++ b/src/lambda-manager/lambda_function.py @@ -204,11 +204,12 @@ def lambda_handler(event, context): print(f"Failed with exception: {e}") status = cfnresponse.FAILED finally: - print("Sending response to custom resource") - cfnresponse.send( - event, - context, - status, - {}, - event.get('PhysicalResourceId', context.aws_request_id) - ) + if "RequestType" in event: + print("Sending response to custom resource") + cfnresponse.send( + event, + context, + status, + {}, + event.get('PhysicalResourceId', context.aws_request_id) + )