[Logging] Add ISO 8601 timestamps to cloud-init logs with milliseconds precision and configure the custom logger for Chef.

CHANGELOG.md

@@ -8,6 +8,7 @@ CHANGELOG
 - Add validator that warns against the downsides of disabling in-place updates on compute and login nodes through DevSettings.
 - Upgrade jmespath to ~=1.0 (from ~=0.10).
 - Upgrade tabulate to <=0.9.0 (from <=0.8.10).
+- Add ISO 8601 timestamps with milliseconds precision to cloud-init-output.log and chef-client.log to facilitate tracing and debugging.
 
 **BUG FIXES**
 - Add validation to block updates that change tag order. Blocking such change prevents update failures.

cli/src/pcluster/config/config_patch.py

@@ -221,11 +221,7 @@ def _compare_list(self, base_section, target_section, param_path, data_key, fiel
         for target_nested_section in target_list:
             update_key_value = target_nested_section.get(update_key)
             base_nested_section = next(
-                (
-                    nested_section
-                    for nested_section in base_list
-                    if nested_section.get(update_key) == update_key_value
-                ),
+                (nested_section for nested_section in base_list if nested_section.get(update_key) == update_key_value),
                 None,
             )
             if base_nested_section:

cli/src/pcluster/resources/compute_node/user_data.sh

@@ -49,7 +49,7 @@ repo_upgrade: none
 datasource_list: [ Ec2, None ]
 
 output:
-  all: "| tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0"
+  all: "| { echo -n \"[$(date '+%Y-%m-%d %H:%M:%S,%3N')] \"; cat; } | tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0"
 write_files:
   - path: /tmp/dna.json
     permissions: '0644'
@@ -65,7 +65,9 @@ write_files:
     permissions: '0744'
     owner: root:root
     content: |
-      #!/bin/bash -x
+      #!/bin/bash
+      PS4="[$(date '+%Y-%m-%d %H:%M:%S,%3N')] "
+      set -x
 
       function error_exit
       {

cli/src/pcluster/resources/head_node/user_data.sh

@@ -56,11 +56,16 @@ repo_upgrade: none
 
 datasource_list: [ Ec2, None ]
 
+output:
+  all: "| { echo -n \"[$(date '+%Y-%m-%d %H:%M:%S,%3N')] \"; cat; } | tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0"
+
 --==BOUNDARY==
 Content-Type: text/x-shellscript; charset="us-ascii"
 MIME-Version: 1.0
 
-#!/bin/bash -x
+#!/bin/bash
+PS4="[$(date '+%Y-%m-%d %H:%M:%S,%3N')] "
+set -x
 
 function error_exit
 {

cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml

@@ -229,6 +229,7 @@ phases:
           - path: /etc/chef/client.rb
             content: |
               cookbook_path ['/etc/chef/cookbooks']
+              require '/etc/chef/cookbooks/aws-parallelcluster-shared/libraries/log_formatter'
             overwrite: true
           - path: /etc/parallelcluster/image_dna.json
             content: |

cli/src/pcluster/resources/login_node/user_data.sh

@@ -46,13 +46,15 @@ repo_upgrade: none
 datasource_list: [ Ec2, None ]
 
 output:
-  all: "| tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0"
+  all: "| { echo -n \"[$(date '+%Y-%m-%d %H:%M:%S,%3N')] \"; cat; } | tee -a /var/log/cloud-init-output.log | logger -t user-data -s 2>/dev/ttyS0"
 write_files:
   - path: /tmp/bootstrap.sh
     permissions: '0744'
     owner: root:root
     content: |
-      #!/bin/bash -x
+      #!/bin/bash
+      PS4="[$(date '+%Y-%m-%d %H:%M:%S,%3N')] "
+      set -x
 
       function error_exit
       {