Skip to content

Commit

Permalink
[202305] Optimize syslog rate limit feature for fast and warm boot (#…
Browse files Browse the repository at this point in the history
…17478)

Backport PR #17458 due to conflict.

Why I did it
Optimize syslog rate limit feature for fast and warm boot

Work item tracking
Microsoft ADO (number only):
How I did it
Optimize redis start time
Don't render rsyslog.conf in container startup script
Disable containercfgd by default. There is a new CLI to enable it (in another PR)
How to verify it
Manual test
Regression test
  • Loading branch information
Junchao-Mellanox authored Dec 28, 2023
1 parent 690c4c5 commit a8ad19d
Show file tree
Hide file tree
Showing 21 changed files with 455 additions and 78 deletions.
2 changes: 1 addition & 1 deletion dockers/docker-base-bullseye/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -119,10 +119,10 @@ RUN apt-get clean -y && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache

COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
COPY ["root/.vimrc", "/root/.vimrc"]

RUN ln /usr/bin/vim.tiny /usr/bin/vim

COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]
78 changes: 78 additions & 0 deletions dockers/docker-base-bullseye/etc/rsyslog.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html


#################
#### MODULES ####
#################

$ModLoad imuxsock # provides support for local system logging

#
# Set a rate limit on messages from the container
#
$SystemLogRateLimitInterval 300
$SystemLogRateLimitBurst 20000

#$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability

# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514

# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514


###########################
#### GLOBAL DIRECTIVES ####
###########################

set $.CONTAINER_NAME=getenv("CONTAINER_NAME");

# Set remote syslog server
template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")

#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Define a custom template
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
$ActionFileDefaultTemplate SONiCFileFormat

#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022

#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

#
# Suppress duplicate messages and report "message repeated n times"
#
$RepeatedMsgReduction on

###############
#### RULES ####
###############
2 changes: 1 addition & 1 deletion dockers/docker-base-buster/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,10 @@ RUN apt-get clean -y && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache/

COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
COPY ["root/.vimrc", "/root/.vimrc"]

RUN ln /usr/bin/vim.tiny /usr/bin/vim

COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]
78 changes: 78 additions & 0 deletions dockers/docker-base-buster/etc/rsyslog.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html


#################
#### MODULES ####
#################

$ModLoad imuxsock # provides support for local system logging

#
# Set a rate limit on messages from the container
#
$SystemLogRateLimitInterval 300
$SystemLogRateLimitBurst 20000

#$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability

# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514

# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514


###########################
#### GLOBAL DIRECTIVES ####
###########################

set $.CONTAINER_NAME=getenv("CONTAINER_NAME");

# Set remote syslog server
template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")

#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Define a custom template
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
$ActionFileDefaultTemplate SONiCFileFormat

#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022

#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

#
# Suppress duplicate messages and report "message repeated n times"
#
$RepeatedMsgReduction on

###############
#### RULES ####
###############
9 changes: 0 additions & 9 deletions dockers/docker-base-buster/etc/supervisor/containercfgd.conf

This file was deleted.

2 changes: 1 addition & 1 deletion dockers/docker-base-stretch/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,10 @@ RUN apt-get clean -y && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/* /tmp/*

COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
COPY ["root/.vimrc", "/root/.vimrc"]

RUN ln /usr/bin/vim.tiny /usr/bin/vim

COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]
78 changes: 78 additions & 0 deletions dockers/docker-base-stretch/etc/rsyslog.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html


#################
#### MODULES ####
#################

$ModLoad imuxsock # provides support for local system logging

#
# Set a rate limit on messages from the container
#
$SystemLogRateLimitInterval 300
$SystemLogRateLimitBurst 20000

#$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability

# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514

# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514


###########################
#### GLOBAL DIRECTIVES ####
###########################

set $.CONTAINER_NAME=getenv("CONTAINER_NAME");

# Set remote syslog server
template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")

#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Define a custom template
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
$ActionFileDefaultTemplate SONiCFileFormat

#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022

#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

#
# Suppress duplicate messages and report "message repeated n times"
#
$RepeatedMsgReduction on

###############
#### RULES ####
###############
9 changes: 0 additions & 9 deletions dockers/docker-base-stretch/etc/supervisor/containercfgd.conf

This file was deleted.

2 changes: 1 addition & 1 deletion dockers/docker-base/Dockerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ RUN apt-get -y install \
rsyslog \
less

COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
COPY ["root/.vimrc", "/root/.vimrc"]

Expand All @@ -67,7 +68,6 @@ RUN mkdir -p /etc/supervisor/conf.d
RUN mkdir -p /var/log/supervisor
COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]
COPY ["etc/supervisor/containercfgd.conf", "/etc/supervisor/conf.d/"]
RUN apt-get -y purge \
exim4 \
Expand Down
78 changes: 78 additions & 0 deletions dockers/docker-base/etc/rsyslog.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html


#################
#### MODULES ####
#################

$ModLoad imuxsock # provides support for local system logging

#
# Set a rate limit on messages from the container
#
$SystemLogRateLimitInterval 300
$SystemLogRateLimitBurst 20000

#$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability

# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514

# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514


###########################
#### GLOBAL DIRECTIVES ####
###########################

set $.CONTAINER_NAME=getenv("CONTAINER_NAME");

# Set remote syslog server
template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")

#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat

# Define a custom template
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
$ActionFileDefaultTemplate SONiCFileFormat

#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022

#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog

#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

#
# Suppress duplicate messages and report "message repeated n times"
#
$RepeatedMsgReduction on

###############
#### RULES ####
###############
Loading

0 comments on commit a8ad19d

Please sign in to comment.